From eeb3fb02c60fce304c1f6b7d5f918a51ff8d4d08 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 23 Oct 2026 14:33:32 -0700 Subject: [PATCH 01/20] Move the Audit Self-Test Matrix into tests/ The inline matrix in spec/audit.py becomes tests/test_audit_helpers.py, a unittest module with one subTest per case. Temporary git repositories are built by setUp-style helpers, and the globals() swaps are mock.patch calls. The invalid-byte probe tests follow the probe out of tests/test_audit.py. --selftest stays as a smoke check of each helper family that runs in well under a second. --- spec/audit.py | 4329 +------------------------------- tests/test_audit.py | 82 +- tests/test_audit_helpers.py | 4311 +++++++++++++++++++++++++++++++ tests/test_fidelity_honesty.py | 2 +- 4 files changed, 4347 insertions(+), 4377 deletions(-) create mode 100755 tests/test_audit_helpers.py diff --git a/spec/audit.py b/spec/audit.py index 934500cde..434dc569b 100755 --- a/spec/audit.py +++ b/spec/audit.py @@ -32,21 +32,16 @@ import argparse import base64 -import contextlib import fnmatch import functools import hashlib -import io import itertools import json -import locale import pathlib import posixpath import re import subprocess import sys -import tempfile -import time import urllib.error import urllib.parse import urllib.request @@ -3777,4307 +3772,49 @@ def _selftest(): def _selftest_cases(): - """Exercise the interface engine on synthetic fixtures, no network - verify the mechanism, not the fleet.""" - pr_check = " check-workflow-status:\n name: Check pull request workflow status job\n needs: [changes]\n runs-on: ubuntu-latest\n" - pr_head = "name: Test\non: pull_request\njobs:\n changes:\n runs-on: ubuntu-latest\n" - pr_contract = { - "requiredJobKeys": ["check-workflow-status"], - "requiredCheckName": "Check pull request workflow status job", - } - # The validate-task.yml stub contract: a caller's validate job must exist and reach the hub task by name. - # A token check alone would pass a stub that drops the validate job entirely, since it only runs for a job that is present. - # Naming the validate job in requiredJobKeys too catches a dropped job on its own. - pr_stub_contract = dict( - pr_contract, - requiredJobKeys=["check-workflow-status", "validate"], - requireTokensInJob={"validate": ["validate-task.yml"]}, - ) - pr_validate_head = ( - "name: Test\non: pull_request\njobs:\n" - " validate:\n" - " name: Validate sources job\n" - " uses: ptr727/ProjectTemplate/.github/workflows/validate-task.yml@" - + "a" * 40 - + " # 2.0.1\n" - ) - pr_validate_inline = ( - "name: Test\non: pull_request\njobs:\n" - " validate:\n" - " name: Validate sources job\n" - " runs-on: ubuntu-latest\n" - " steps:\n - run: echo inline lint\n" - ) - # A validate-task.yml stub's own aggregator needs the validate job, never the release-shape changes job. - # The shared pr_check fixture needs a changes job these two fixtures do not define, so this one needs validate instead. - pr_check_validate = " check-workflow-status:\n name: Check pull request workflow status job\n needs: [validate]\n runs-on: ubuntu-latest\n" - gh_rel = ( - " github-release:\n needs: [get-version, build-widget]\n runs-on: ubuntu-latest\n steps:\n" - " - uses: actions/download-artifact@v4\n with:\n pattern: release-asset-${{ inputs.branch }}-*\n merge-multiple: true\n" - ) - rel_head = ( - "name: Build Release\non:\n workflow_call:\njobs:\n" - " get-version:\n runs-on: ubuntu-latest\n steps: []\n" - " build-widget:\n runs-on: ubuntu-latest\n steps: []\n" - ) - rel_ok = rel_head + gh_rel - rel_contract = { - "requiredJobKeys": ["get-version", "github-release"], - "artifactNameToken": "release-asset-", - "requireTokensInJob": {"github-release": ["pattern:", "merge-multiple:"]}, - "forbidTokensInJob": {"github-release": ["artifact-ids:"]}, - } - # The merge-bot caller stub, the shape every repo carries once the merge-bot is hub-hosted. - bot_stub = ( - "jobs:\n" - " merge-bot:\n" - " name: Merge bot pull request job\n" - " uses: acme/hub/.github/workflows/merge-bot-task.yml@" + "a" * 40 + " # 2.0.1\n" - " secrets:\n" - " CODEGEN_APP_CLIENT_ID: ${{ secrets.CODEGEN_APP_CLIENT_ID }}\n" - " CODEGEN_APP_PRIVATE_KEY: ${{ secrets.CODEGEN_APP_PRIVATE_KEY }}\n" - ) - bot_contract = { - "requiredJobKeys": ["merge-bot"], - "requireTokensInJob": { - "merge-bot": ["merge-bot-task.yml", "CODEGEN_APP_CLIENT_ID", "CODEGEN_APP_PRIVATE_KEY"] - }, - } - # The publish-release.yml caller stub once the release chain is hub-hosted, plan, validate and publish job keys. - # Each names its hub task by token, per docs/reusable-workflows.md "Adopting the Release Chain". - # Validate's own token requirement is what stops a stub from satisfying the job key with a no-op job. - publish_stub = ( - "jobs:\n" - " plan:\n" - " name: Plan release job\n" - " uses: acme/hub/.github/workflows/publish-plan-task.yml@" + "a" * 40 + " # 2.0.1\n" - " with:\n" - " event_name: ${{ github.event_name }}\n" - " validate:\n" - " name: Validate sources job\n" - " needs: [plan]\n" - " if: ${{ needs.plan.outputs.publish == 'true' }}\n" - " uses: ./.github/workflows/validate-task.yml\n" - " publish:\n" - " name: Publish project release job\n" - " needs: [plan, validate]\n" - " if: ${{ needs.plan.outputs.publish == 'true' && needs.validate.result == 'success' }}\n" - " uses: acme/hub/.github/workflows/build-release-task.yml@" + "a" * 40 + " # 2.0.1\n" - " with:\n" - " github: true\n" - ) - publish_contract = { - "requiredJobKeys": ["plan", "validate", "publish"], - "requireTokensInJob": { - "plan": ["publish-plan-task.yml"], - "validate": ["validate-task.yml"], - "publish": ["build-release-task.yml", "needs.validate.result == 'success'"], - }, - } + """A smoke check of each helper family on constructed fixtures; tests/test_audit_helpers.py holds the suite.""" + pr = "name: T\non: pull_request\njobs:\n check:\n name: Check job\n runs-on: ubuntu-latest\n" + contract = {"requiredJobKeys": ["check"], "requiredCheckName": "Check job"} + publisher = ( + "on:\n push:\nconcurrency:\n group: ${{ github.workflow }}-${{ github.ref }}\n" + " cancel-in-progress: false\njobs:\n validate:\n uses: o/h/.github/workflows/validate-task.yml@x\n" + " with:\n python-directories: |\n scripts\n" + ) + doc = "# T\n\nTagline [link](https://example.invalid).\n\n## A\n\n```\n## B\n```\n\n## C\n" cases = [ - ("conformant PR workflow", pr_head + pr_check, pr_contract, 0), - ("PR workflow missing the required job and its check name", pr_head, pr_contract, 2), - ( - "PR workflow with a renamed check", - pr_head + pr_check.replace("Check pull request workflow status job", "Renamed"), - pr_contract, - 1, - ), - ( - "check name present only in a run step, not as a job name", - pr_head - + " check-workflow-status:\n runs-on: ubuntu-latest\n steps:\n - run: echo Check pull request workflow status job\n", - pr_contract, - 1, - ), - ( - "check name present only as a step name, not the job name", - pr_head - + " check-workflow-status:\n runs-on: ubuntu-latest\n steps:\n - name: Check pull request workflow status job\n run: true\n", - pr_contract, - 1, - ), - ( - "PR stub validate job reaching the hub validate-task", - pr_validate_head + pr_check_validate, - pr_stub_contract, - 0, - ), - ( - "PR stub validate job still carrying an inline lint job", - pr_validate_inline + pr_check_validate, - pr_stub_contract, - 1, - ), - ( - "PR stub dropping the validate job entirely", - pr_head + pr_check, - pr_stub_contract, - 1, - ), - ("conformant release task", rel_ok, rel_contract, 0), - ( - "release task with an artifact-ids fork in github-release", - rel_ok.replace( - " merge-multiple: true\n", - " merge-multiple: true\n artifact-ids: 123\n", - ), - rel_contract, - 1, - ), - ( - "release task missing merge-multiple in github-release", - rel_ok.replace(" merge-multiple: true\n", ""), - rel_contract, - 1, - ), - ( - "release task with an owned extra leaf job", - rel_head + " build-extra:\n runs-on: ubuntu-latest\n steps: []\n" + gh_rel, - rel_contract, - 0, - ), - ( - "absent job reports once, no redundant token findings", - rel_head, - { - "requiredJobKeys": ["github-release"], - "requireTokensInJob": {"github-release": ["pattern:", "merge-multiple:"]}, - }, - 1, - ), - ( - "a forbidden token only in a comment is ignored", - rel_ok.replace( - " merge-multiple: true\n", - " merge-multiple: true\n # never an artifact-ids: fork here\n", - ), - rel_contract, - 0, - ), - ( - "a required token only in a comment does not count", - rel_ok.replace( - " merge-multiple: true\n", " # merge-multiple: true (was here)\n" - ), - rel_contract, - 1, - ), - ( - "merge-bot caller stub reaching the hub task with both secrets mapped", - bot_stub, - bot_contract, - 0, - ), - ( - "merge-bot caller stub with the hub's own local uses", - bot_stub.replace( - "acme/hub/.github/workflows/merge-bot-task.yml@" + "a" * 40 + " # 2.0.1", - "./.github/workflows/merge-bot-task.yml", - ), - bot_contract, - 0, - ), - ( - "merge-bot copy still carrying the job bodies reports the missing caller job once", - ( - "jobs:\n merge-dependabot:\n runs-on: ubuntu-latest\n steps: []\n" - " disable-auto-merge-on-maintainer-push:\n runs-on: ubuntu-latest\n steps: []\n" - ), - bot_contract, - 1, - ), - ( - "merge-bot caller stub that maps only one secret", - bot_stub.replace( - " CODEGEN_APP_PRIVATE_KEY: ${{ secrets.CODEGEN_APP_PRIVATE_KEY }}\n", "" - ), - bot_contract, - 1, - ), - ( - "publish-release.yml stub reaching both hub tasks by pin", - publish_stub, - publish_contract, - 0, - ), - ( - "publish-release.yml stub with the hub's own local uses", - publish_stub.replace( - "acme/hub/.github/workflows/publish-plan-task.yml@" + "a" * 40 + " # 2.0.1", - "./.github/workflows/publish-plan-task.yml", - ).replace( - "acme/hub/.github/workflows/build-release-task.yml@" + "a" * 40 + " # 2.0.1", - "./.github/workflows/build-release-task.yml", - ), - publish_contract, - 0, - ), - ( - "publish-release.yml stub missing the plan job reports it once", - "jobs:\n" - " validate:\n" - " name: Validate sources job\n" - " uses: ./.github/workflows/validate-task.yml\n" - " publish:\n" - " name: Publish project release job\n" - " needs: [validate]\n" - " if: ${{ needs.validate.result == 'success' }}\n" - " uses: acme/hub/.github/workflows/build-release-task.yml@" - + "a" * 40 - + " # 2.0.1\n", - publish_contract, - 1, - ), - ( - "publish-release.yml stub missing the validate job reports it once", - "jobs:\n" - " plan:\n" - " name: Plan release job\n" - " uses: acme/hub/.github/workflows/publish-plan-task.yml@" + "a" * 40 + " # 2.0.1\n" - " publish:\n" - " name: Publish project release job\n" - " needs: [plan, validate]\n" - " if: ${{ needs.validate.result == 'success' }}\n" - " uses: acme/hub/.github/workflows/build-release-task.yml@" - + "a" * 40 - + " # 2.0.1\n", - publish_contract, - 1, - ), - ( - "publish-release.yml stub whose publish job never names build-release-task.yml", - publish_stub.replace( - "acme/hub/.github/workflows/build-release-task.yml@" + "a" * 40 + " # 2.0.1", - "./.github/workflows/build-release-task-renamed.yml", - ), - publish_contract, - 1, - ), - ( - "publish-release.yml stub whose validate job never names validate-task.yml", - publish_stub.replace( - " uses: ./.github/workflows/validate-task.yml\n", - " uses: ./.github/workflows/validate-task-renamed.yml\n", - ), - publish_contract, - 1, - ), - ( - "publish-release.yml stub carrying its publish condition in a folded if: >-", - publish_stub.replace( - " if: ${{ needs.plan.outputs.publish == 'true' && needs.validate.result == 'success' }}\n", - " if: >-\n" - " ${{ needs.plan.outputs.publish == 'true'\n" - " && needs.validate.result == 'success' }}\n", - ), - publish_contract, - 0, - ), - ( - "publish-release.yml stub whose folded if: >- wraps inside the required token", - publish_stub.replace( - " if: ${{ needs.plan.outputs.publish == 'true' && needs.validate.result == 'success' }}\n", - " if: >-\n" - " ${{ needs.plan.outputs.publish == 'true' && needs.validate.result\n" - " == 'success' }}\n", - ), - publish_contract, - 0, - ), - ( - "publish-release.yml stub naming the validation token only in a name: | body", - publish_stub.replace( - " name: Publish project release job\n", - " name: |\n needs.validate.result == 'success'\n", - ).replace(" && needs.validate.result == 'success'", ""), - publish_contract, - 1, - ), - ( - "publish-release.yml stub whose publish job ignores failed validation", - publish_stub.replace(" && needs.validate.result == 'success'", ""), - publish_contract, - 1, - ), - ] - # The deploy-site.yml caller stub once deploy-site-task.yml is hub-hosted: no secrets: inherit, the required crossing secret named instead. - # inherit is documented for same-org or enterprise callers, so the fleet does not use it across repositories. - # No job-level environment: on the caller, unsupported on a job with uses:. - deploy_stub = ( - "jobs:\n" - " assert-ref:\n runs-on: ubuntu-latest\n steps: []\n" - " validate:\n uses: ./.github/workflows/validate-task.yml\n" - " deploy:\n" - " name: Deploy job\n" - " permissions:\n contents: read\n" - " uses: acme/hub/.github/workflows/deploy-site-task.yml@" + "a" * 40 + " # 2.0.1\n" - " with:\n environment: ${{ inputs.environment }}\n" - " secrets:\n DEPLOY_SSH_PRIVATE_KEY: ${{ secrets.DEPLOY_SSH_PRIVATE_KEY }}\n" - ) - deploy_contract = { - "requiredJobKeys": ["assert-ref", "validate", "deploy"], - "requireTokensInJob": { - "deploy": [ - "deploy-site-task.yml", - # Anchored to the with: block specifically, not just 6-space indent. - # A secret named "environment" would also land at that indent otherwise. - # The task declares this input required, and a missing one fails at dispatch, not at audit time. - "with:\n environment:", - "contents: read", - "DEPLOY_SSH_PRIVATE_KEY", - ] - }, - "forbidTokensInJob": { - # Indent-anchored (4 spaces) to the job's own top level, the shape GitHub rejects - # at parse time on a job with uses:. - "deploy": ["\n environment:"] - }, - } - cases += [ - ( - "deploy-site.yml caller stub reaching the hub task with the crossing secret mapped", - deploy_stub, - deploy_contract, - 0, - ), - ( - "deploy-site.yml caller stub still using secrets: inherit reports the missing secret", - deploy_stub.replace( - " secrets:\n DEPLOY_SSH_PRIVATE_KEY: ${{ secrets.DEPLOY_SSH_PRIVATE_KEY }}\n", - " secrets: inherit\n", - ), - deploy_contract, - 1, - ), - ( - "deploy-site.yml caller stub reintroducing the invalid job-level environment: key", - deploy_stub.replace( - " name: Deploy job\n", - " name: Deploy job\n environment: ${{ inputs.environment }}\n", - ), - deploy_contract, - 1, - ), - ( - "deploy-site.yml caller stub missing the required environment input", - deploy_stub.replace(" with:\n environment: ${{ inputs.environment }}\n", ""), - deploy_contract, - 1, - ), - ( - "deploy-site.yml caller stub with environment: under the wrong mapping still reports missing", - # A same-indented environment: key under secrets: must not satisfy the with:-anchored requirement on its own. - # The oddly-named secret's value is left generic so it does not incidentally satisfy the DEPLOY_SSH_PRIVATE_KEY token. - deploy_stub.replace( - " with:\n environment: ${{ inputs.environment }}\n", - " secrets:\n environment: ${{ secrets.SOME_OTHER_SECRET }}\n", - ), - deploy_contract, - 1, - ), + ("a conformant workflow", len(check_interface("wf", contract, pr)), 0), + ("a workflow missing its job", len(check_interface("wf", contract, "jobs: {}\n")), 2), ( - "deploy-site.yml caller stub missing the contents: read grant the task's jobs declare", - deploy_stub.replace(" permissions:\n contents: read\n", ""), - deploy_contract, - 1, + "a pinned concurrency group", + concurrency_caller_findings(".github/workflows/publish-release.yml", publisher), + [], ), ( - # A block scalar crafted to contain the required token as string content must still report it missing. - # The real `with:` mapping is removed here, so only the block scalar carries the text. - "deploy-site.yml caller stub with the with:/environment: tokens only inside a block-scalar name still reports missing", - deploy_stub.replace( - " name: Deploy job\n", " name: |\n with:\n environment:\n" - ).replace(" with:\n environment: ${{ inputs.environment }}\n", ""), - deploy_contract, - 1, - ), - ] - # The stage-5 type-specific hub tasks carry no manifest entry yet, since no repo has adopted a caller stub for them. - # These fixtures exercise the contract adoption will register, proving the interface engine reads it correctly before any downstream repo depends on that reading. - readme_stub = ( - "jobs:\n" - " publish-docker-readme:\n" - " name: Publish Docker Hub readme job\n" - " permissions:\n contents: read\n" - " uses: acme/hub/.github/workflows/publish-docker-readme-task.yml@" - + "a" - * 40 - + " # 2.0.1\n" - " with:\n branch: ${{ github.ref_name }}\n" - " secrets:\n" - " DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}\n" - " DOCKER_HUB_ACCESS_TOKEN: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}\n" - ) - readme_contract = { - "requiredJobKeys": ["publish-docker-readme"], - "requireTokensInJob": { - "publish-docker-readme": [ - "publish-docker-readme-task.yml", - "contents: read", - "DOCKER_HUB_USERNAME", - "DOCKER_HUB_ACCESS_TOKEN", - ] - }, - } - upstream_stub = ( - "jobs:\n" - " check-upstream-version:\n" - " name: Check upstream version job\n" - " uses: acme/hub/.github/workflows/check-upstream-version-task.yml@" - + "a" - * 40 - + " # 2.0.1\n" - " secrets:\n" - " CODEGEN_APP_CLIENT_ID: ${{ secrets.CODEGEN_APP_CLIENT_ID }}\n" - " CODEGEN_APP_PRIVATE_KEY: ${{ secrets.CODEGEN_APP_PRIVATE_KEY }}\n" - ) - upstream_contract = { - "requiredJobKeys": ["check-upstream-version"], - "requireTokensInJob": { - "check-upstream-version": [ - "check-upstream-version-task.yml", - "CODEGEN_APP_CLIENT_ID", - "CODEGEN_APP_PRIVATE_KEY", - ] - }, - } - codegen_stub = ( - "jobs:\n" - " run-codegen:\n" - " name: Run codegen and pull request job\n" - " uses: acme/hub/.github/workflows/run-codegen-pull-request-task.yml@" - + "a" - * 40 - + " # 2.0.1\n" - " secrets:\n" - " CODEGEN_APP_CLIENT_ID: ${{ secrets.CODEGEN_APP_CLIENT_ID }}\n" - " CODEGEN_APP_PRIVATE_KEY: ${{ secrets.CODEGEN_APP_PRIVATE_KEY }}\n" - ) - codegen_contract = { - "requiredJobKeys": ["run-codegen"], - "requireTokensInJob": { - "run-codegen": [ - "run-codegen-pull-request-task.yml", - "CODEGEN_APP_CLIENT_ID", - "CODEGEN_APP_PRIVATE_KEY", - ] - }, - } - for label_prefix, stub, contract in ( - ("publish-docker-readme.yml", readme_stub, readme_contract), - ("check-upstream-version.yml", upstream_stub, upstream_contract), - ("run-codegen-pull-request.yml", codegen_stub, codegen_contract), - ): - cases += [ - ( - f"{label_prefix} caller stub reaching the hub task with secrets mapped", - stub, - contract, - 0, - ), - ( - f"{label_prefix} copy still carrying the job body reports the missing caller job", - "jobs:\n some-other-job:\n runs-on: ubuntu-latest\n steps: []\n", - contract, - 1, + "a declared python directory", + python_directories_caller_findings( + ".github/workflows/publish-release.yml", + publisher, + {"types": ["python"], "pythonDirectories": ["scripts"]}, ), - ] - ok = True - for label, text, contract, want in cases: - got = len(check_interface("wf", contract, text)) - if got != want: - ok = False - print(f" {'ok ' if got == want else 'FAIL'} want={want} got={got} {label}") - trailing = split_jobs(rel_ok + "# a trailing top-level comment\n") - if set(trailing) != { - "get-version", - "build-widget", - "github-release", - } or "trailing top-level comment" in trailing.get("github-release", ""): - ok = False - print(f" FAIL split_jobs (trailing comment) -> {sorted(trailing)}") - else: - print(f" ok split_jobs (trailing comment) -> {sorted(trailing)}") - inline = split_jobs( - "name: X\non: push\njobs:\n quick: {runs-on: ubuntu-latest}\n full:\n runs-on: ubuntu-latest\n" - ) - if set(inline) != {"quick", "full"} or "runs-on" not in inline.get("quick", ""): - ok = False - print(f" FAIL split_jobs (inline mapping) -> {sorted(inline)}") - else: - print(" ok split_jobs (inline-mapping job captured with its content)") - - # _code_view()'s block-scalar handling: only the body is dropped, never the key line itself. - code_view_cases = [ - ( - "block scalar body dropped, key line kept", - " deploy:\n name: |\n with:\n environment:\n", - " deploy:\n name: |", - ), - ( - "a dedented sibling key ends the block scalar body", - " deploy:\n name: |\n with:\n with:\n environment: prod\n", - " deploy:\n name: |\n with:\n environment: prod", - ), - ( - "folded scalar, strip-chomp indicator, trailing comment on the key line", - " deploy:\n name: >- # a folded, strip-chomped scalar\n environment:\n", - " deploy:\n name: >- # a folded, strip-chomped scalar", - ), - ( - # A step's `- run: |` boundary is `run:`'s column, not the dash's column. - # A sibling `env:` key genuinely at that column is real structure, not block-scalar body. - "a step's `- run: |` block scalar body drops, its sibling env: mapping survives", - " - run: |\n echo body\n env:\n TOKEN: xyz\n", - " - run: |\n env:\n TOKEN: xyz", - ), - ( - # A valid YAML anchor property (`&label`) sits between the colon and the indicator. - "an anchored block scalar (`name: &label |`) is still recognized as a block-scalar key", - " deploy:\n name: &lbl |\n with:\n environment:\n", - " deploy:\n name: &lbl |", - ), - ( - # A bare sequence item (a matrix string) has no key, so its boundary is the dash's column. - "a bare sequence-item block scalar (`- |`, a matrix string) still drops its body", - " include:\n - |\n TOKEN\n", - " include:\n - |", - ), - ( - "a folded `if: >-` keeps its body, folded onto the key line", - " publish:\n if: >-\n ${{ a\n\n && b }}\n uses: x\n", - " publish:\n if: >- ${{ a && b }}\n uses: x", - ), - ( - "a step's folded `- if: >-` keeps its body too", - " - if: >-\n ${{ a }}\n run: echo\n", - " - if: >- ${{ a }}\n run: echo", - ), - ] - for label, text, want in code_view_cases: - got = _code_view(text) - if got != want: - ok = False - print(f" FAIL _code_view ({label}) -> {got!r}, want {want!r}") - else: - print(f" ok _code_view ({label})") - - # The verbatim engine, covering EOL normalization, hashing, and the stale-versus-modified classification. - # It is exercised here rather than only in production, because a latent bug in the comparison would otherwise surface as a false clean on a real fleet run. - canon = "line one\nline two\nline three\n" - verbatim_cases = [ - # (label, down_text, canon_text, history, want) - ("identical -> match", canon, canon, [], None), - ("EOL-only diff (CRLF) -> match", canon.replace("\n", "\r\n"), canon, [], None), - ("EOL-only diff (bare CR) -> match", canon.replace("\n", "\r"), canon, [], None), - ( - "body edit -> modified", - canon.replace("line two", "line TWO edited"), - canon, [], - "modified", - ), - ( - "matches a past revision -> stale", - "old body\n", - "current body\n", - ["old body\n", "older\n"], - "stale", - ), - ( - "matches a past revision modulo EOL -> stale", - "old body\r\n", - "current body\n", - ["old body\n"], - "stale", - ), - ( - "edit in no revision -> modified", - "never existed\n", - "current body\n", - ["old body\n"], - "modified", - ), - ] - for label, down, canon_t, history, want in verbatim_cases: - got = classify_verbatim(down, canon_t, history) - if got != want: - ok = False - print( - f" {'ok ' if got == want else 'FAIL'} want={want!s:>8} got={got!s:>8} verbatim: {label}" - ) - # Action-pin neutralization, where a Dependabot uses:@ bump, meaning both the 40-hex sha and its ` # vN` comment, must not count as verbatim drift, while a changed action name must. - # This is what lets a verbatim workflow region survive routine action bumps while still catching a real fork. - pin_a = " - uses: actions/checkout@" + "a" * 40 + " # v7.0.0\n" - pin_b = ( - " - uses: actions/checkout@" + "B" * 40 + " # v7.0.1\n" - ) # uppercase hex + version bump - pin_struct = " - uses: actions/setup-node@" + "a" * 40 + " # v7.0.0\n" - note_x = " - uses: actions/checkout@" + "a" * 40 + " # kept for the audited build\n" - note_y = " - uses: actions/checkout@" + "a" * 40 + " # kept for a different reason\n" - if content_hash(pin_a) != content_hash(pin_b): - ok = False - print( - " FAIL action-pin: a uses:@ bump (sha + # vN, uppercase hex) should normalize equal" - ) - elif content_hash(pin_a) == content_hash(pin_struct): - ok = False - print(" FAIL action-pin: a changed action name must still hash differently") - elif content_hash(note_x) == content_hash(note_y): - ok = False - print( - " FAIL action-pin: a hand-written (non-version) pin comment must survive to be compared" - ) - else: - print( - " ok action-pin: version bump normalizes equal, changed action differs, hand-written note survives" - ) - - # The needs-mask case, where a verbatim job region whose `needs:` list is pruned to the repo's vendored targets must not count as drift, since the list is owned, while a structural change to the job's steps must. - needs_full = " github-release:\n needs: [get-version, validate-release, build-nuget, dotnet-publish]\n runs-on: x\n steps: []\n" - needs_pruned = " github-release:\n needs: [get-version, validate-release, dotnet-publish]\n runs-on: x\n steps: []\n" - needs_block = " github-release:\n needs:\n - get-version\n - dotnet-publish\n runs-on: x\n steps: []\n" - needs_scalar = " github-release:\n needs: dotnet-publish\n runs-on: x\n steps: []\n" - needs_forked = " github-release:\n needs: [get-version, validate-release]\n runs-on: x\n steps:\n - run: fork\n" - if ( - len( - { - content_hash(needs_full), - content_hash(needs_pruned), - content_hash(needs_block), - content_hash(needs_scalar), - } - ) - != 1 - ): - ok = False - print( - " FAIL needs-mask: a pruned needs list (inline, block, or scalar) should normalize equal" - ) - elif content_hash(needs_full) == content_hash(needs_forked): - ok = False - print(" FAIL needs-mask: a step change must still hash differently") - elif "runs-on: x" not in normalize(needs_block): - ok = False - print(" FAIL needs-mask: masking a block needs list must not consume the next key") - else: - print( - " ok needs-mask: pruned needs (inline, block, scalar) normalizes equal, forked step differs, next key preserved" - ) - - # _last_effective_change must skip a normalized-only bump. - d3, d2, d1 = ( - "2024-03-01T00:00:00+00:00", - "2024-02-01T00:00:00+00:00", - "2024-01-01T00:00:00+00:00", - ) - lec_cases = [ - ( - "every bump back to creation is pin-only -> creation wins", - [(d3, "sha3", pin_b), (d2, "sha2", pin_a), (d1, "sha1", pin_a)], - (d1, "sha1"), - ), - ( - "newest differs for real -> newest wins", - [(d3, "sha3", pin_struct), (d2, "sha2", pin_a), (d1, "sha1", pin_a)], - (d3, "sha3"), ), ( - "newest is a pin-only bump over a real change -> the real change wins", - [(d3, "sha3", pin_b), (d2, "sha2", pin_a), (d1, "sha1", pin_struct)], - (d2, "sha2"), + "a fenced heading is not a boundary", + extract_section(doc, "A"), + "## A\n\n```\n## B\n```\n", ), - ( - "one revision (creation) -> that revision wins, nothing to compare against", - [(d1, "sha1", "only revision\n")], - (d1, "sha1"), - ), - ( - "unreadable newest text -> treated as effective, never silently skipped", - [(d2, "sha2", None), (d1, "sha1", "whatever\n")], - (d2, "sha2"), - ), - ("no history -> None", [], None), + ("a stale verbatim copy", classify_verbatim("v1\n", "v2\n", ["v1\r\n"]), "stale"), + ("a link reduced to its text", strip_md_links("[a](u) b"), "a b"), + ("the title and intro", title_and_intro(doc)[0], "T"), + ("an undeclared heading", undeclared_h2_headings(doc, ["a"]), ["c"]), + ("a version literal", version_literals_outside_verbatim("uses 1.2.3\n", []), ["1.2.3"]), + ("a secret-file stem", len(secret_stem_findings({".secrets/Bad.txt"})), 1), + ("the selftest flag", parse_args(["--selftest"]).selftest, True), ] - for label, revisions, want in lec_cases: - got = _last_effective_change(revisions) - if got != want: - ok = False - print( - f" {'ok ' if got == want else 'FAIL'} want={want!s:<24} got={got!s:<24} _last_effective_change: {label}" - ) - - # _git_revisions: a deletion revision reads as None even once rel_path exists again in a - # later commit. - with tempfile.TemporaryDirectory() as tmp_root: - tmp_root_path = pathlib.Path(tmp_root) - for cmd in ( - ["git", "init", "-q"], - ["git", "config", "user.email", "test@test.invalid"], - ["git", "config", "user.name", "test"], - ): - subprocess.run(cmd, cwd=tmp_root_path, check=True, capture_output=True) - rel = "deletion-probe.txt" - (tmp_root_path / rel).write_text("v1\n") - subprocess.run(["git", "add", rel], cwd=tmp_root_path, check=True, capture_output=True) - subprocess.run( - ["git", "commit", "-q", "-m", "add"], cwd=tmp_root_path, check=True, capture_output=True - ) - subprocess.run(["git", "rm", "-q", rel], cwd=tmp_root_path, check=True, capture_output=True) - subprocess.run( - ["git", "commit", "-q", "-m", "delete"], - cwd=tmp_root_path, - check=True, - capture_output=True, - ) - (tmp_root_path / rel).write_text("v2\n") - subprocess.run(["git", "add", rel], cwd=tmp_root_path, check=True, capture_output=True) - subprocess.run( - ["git", "commit", "-q", "-m", "readd"], - cwd=tmp_root_path, - check=True, - capture_output=True, - ) - global ROOT - saved_root = ROOT - ROOT = tmp_root_path - try: - _git_revisions.cache_clear() - revisions = _git_revisions(rel, rev="HEAD") - finally: - ROOT = saved_root - _git_revisions.cache_clear() - got = [text for _, _, text in revisions] - want = ["v2\n", None, "v1\n"] - if got != want: - ok = False - print( - f" {'ok ' if got == want else 'FAIL'} want={want!s:<24} got={got!s:<24} _git_revisions: re-added file" - ) - - # _git_revisions: a path that becomes a directory reads as None too. - # git show on a tree path returns a listing rather than file content. - with tempfile.TemporaryDirectory() as tmp_root: - tmp_root_path = pathlib.Path(tmp_root) - for cmd in ( - ["git", "init", "-q"], - ["git", "config", "user.email", "test@test.invalid"], - ["git", "config", "user.name", "test"], - ): - subprocess.run(cmd, cwd=tmp_root_path, check=True, capture_output=True) - rel = "thing" - (tmp_root_path / rel).write_text("v1\n") - subprocess.run(["git", "add", rel], cwd=tmp_root_path, check=True, capture_output=True) - subprocess.run( - ["git", "commit", "-q", "-m", "file"], - cwd=tmp_root_path, - check=True, - capture_output=True, - ) - subprocess.run(["git", "rm", "-q", rel], cwd=tmp_root_path, check=True, capture_output=True) - (tmp_root_path / rel).mkdir() - (tmp_root_path / rel / "inner.txt").write_text("v2\n") - subprocess.run(["git", "add", rel], cwd=tmp_root_path, check=True, capture_output=True) - subprocess.run( - ["git", "commit", "-q", "-m", "now a directory"], - cwd=tmp_root_path, - check=True, - capture_output=True, - ) - saved_root = ROOT - ROOT = tmp_root_path - try: - _git_revisions.cache_clear() - revisions = _git_revisions(rel, rev="HEAD") - finally: - ROOT = saved_root - _git_revisions.cache_clear() - got = [text for _, _, text in revisions] - want = [None, "v1\n"] - if got != want: - ok = False - print( - f" {'ok ' if got == want else 'FAIL'} want={want!s:<24} got={got!s:<24} _git_revisions: file-to-directory transition" - ) - - # _git_revisions: a path that becomes a symlink reads as None too. - # A symlink's ls-tree type is "blob", and git show returns its target path rather than file content. - with tempfile.TemporaryDirectory() as tmp_root: - tmp_root_path = pathlib.Path(tmp_root) - for cmd in ( - ["git", "init", "-q"], - ["git", "config", "user.email", "test@test.invalid"], - ["git", "config", "user.name", "test"], - ): - subprocess.run(cmd, cwd=tmp_root_path, check=True, capture_output=True) - rel = "thing" - (tmp_root_path / rel).write_text("v1\n") - subprocess.run(["git", "add", rel], cwd=tmp_root_path, check=True, capture_output=True) - subprocess.run( - ["git", "commit", "-q", "-m", "file"], - cwd=tmp_root_path, - check=True, - capture_output=True, - ) - subprocess.run(["git", "rm", "-q", rel], cwd=tmp_root_path, check=True, capture_output=True) - try: - (tmp_root_path / rel).symlink_to("target") - except OSError: - # Creating a symlink needs a privilege this host or user may not have (notably - # Windows without Developer Mode or an elevated prompt): skip this case rather than - # aborting the whole --selftest run over an environment limitation, not a code fault. - print(" skip _git_revisions: file-to-symlink transition (host cannot create symlinks)") - else: - subprocess.run(["git", "add", rel], cwd=tmp_root_path, check=True, capture_output=True) - subprocess.run( - ["git", "commit", "-q", "-m", "now a symlink"], - cwd=tmp_root_path, - check=True, - capture_output=True, - ) - saved_root = ROOT - ROOT = tmp_root_path - try: - _git_revisions.cache_clear() - revisions = _git_revisions(rel, rev="HEAD") - finally: - ROOT = saved_root - _git_revisions.cache_clear() - got = [text for _, _, text in revisions] - want = [None, "v1\n"] - if got != want: - ok = False - print( - f" {'ok ' if got == want else 'FAIL'} want={want!s:<24} got={got!s:<24} _git_revisions: file-to-symlink transition" - ) - - # _git_revisions: the default rev reads origin's `main`, not ROOT's checked-out branch. - # `origin` is a plain local path here, so the fetch stays offline. - with tempfile.TemporaryDirectory() as tmp_upstream, tempfile.TemporaryDirectory() as tmp_root: - tmp_upstream_path = pathlib.Path(tmp_upstream) - tmp_root_path = pathlib.Path(tmp_root) - rel = "hub-probe.txt" - for cmd in ( - ["git", "init", "-q", "-b", "main"], - ["git", "config", "user.email", "test@test.invalid"], - ["git", "config", "user.name", "test"], - ): - subprocess.run(cmd, cwd=tmp_upstream_path, check=True, capture_output=True) - (tmp_upstream_path / rel).write_text("main-content\n") - subprocess.run(["git", "add", rel], cwd=tmp_upstream_path, check=True, capture_output=True) - subprocess.run( - ["git", "commit", "-q", "-m", "add on main"], - cwd=tmp_upstream_path, - check=True, - capture_output=True, - ) - subprocess.run( - ["git", "clone", "-q", str(tmp_upstream_path), str(tmp_root_path)], - check=True, - capture_output=True, - ) - # A clone carries no committer identity of its own (no global config on a CI runner either), so this repo needs the same setup as tmp_upstream_path above. - for cmd in ( - ["git", "config", "user.email", "test@test.invalid"], - ["git", "config", "user.name", "test"], - ): - subprocess.run(cmd, cwd=tmp_root_path, check=True, capture_output=True) - subprocess.run( - ["git", "checkout", "-q", "-b", "develop"], - cwd=tmp_root_path, - check=True, - capture_output=True, - ) - (tmp_root_path / rel).write_text("develop-only-content\n") - subprocess.run(["git", "add", rel], cwd=tmp_root_path, check=True, capture_output=True) - subprocess.run( - ["git", "commit", "-q", "-m", "develop-only change"], - cwd=tmp_root_path, - check=True, - capture_output=True, - ) - saved_root = ROOT - ROOT = tmp_root_path - try: - _git_revisions.cache_clear() - _hub_main_rev.cache_clear() - revisions = _git_revisions(rel) - finally: - ROOT = saved_root - _git_revisions.cache_clear() - _hub_main_rev.cache_clear() - got = [text for _, _, text in revisions] - want = ["main-content\n"] - if got != want: - ok = False - print( - f" {'ok ' if got == want else 'FAIL'} want={want!s:<24} got={got!s:<24} _git_revisions: default rev reads origin's main, not the checked-out branch" - ) - - # hub_tracked(): a tracked filename with a byte the active locale rejects must round-trip rather than crash. - # git ls-tree -z is NUL-delimited raw bytes, and decoding it as text before the NUL-split (the bug review caught) raises UnicodeDecodeError instead of enumerating the path. - encoding = locale.getpreferredencoding(False) - invalid_bytes = None - for candidate in (b"\xff", b"\x80\x81", b"\xfe\xff"): - try: - candidate.decode(encoding) - except UnicodeDecodeError: - invalid_bytes = candidate - break - if invalid_bytes is None: - # No candidate is actually invalid under this host's active encoding: skip rather than asserting a regression the fixture cannot exercise here. - print(f" skip hub_tracked: no candidate byte sequence is invalid under {encoding!r}") - else: - bad_name = "bad-" + invalid_bytes.decode("utf-8", errors="surrogateescape") + "-name.txt" - with tempfile.TemporaryDirectory() as tmp_root: - tmp_root_path = pathlib.Path(tmp_root) - for cmd in ( - ["git", "init", "-q", "-b", "main"], - ["git", "config", "user.email", "test@test.invalid"], - ["git", "config", "user.name", "test"], - ): - subprocess.run(cmd, cwd=tmp_root_path, check=True, capture_output=True) - (tmp_root_path / "control.txt").write_text("x") - subprocess.run( - ["git", "add", "--", "control.txt"], - cwd=tmp_root_path, - check=True, - capture_output=True, - ) - added = False - try: - (tmp_root_path / bad_name).write_text("x") - except OSError: - # This host's filesystem cannot represent the byte: skip rather than aborting the whole --selftest run over an environment limitation, not a code fault. - print(" skip hub_tracked: non-UTF-8 filename (host cannot create it)") - else: - try: - subprocess.run( - ["git", "add", "--", bad_name], - cwd=tmp_root_path, - check=True, - capture_output=True, - ) - except subprocess.CalledProcessError: - if not _IS_WINDOWS: - raise - print(" skip hub_tracked: non-UTF-8 filename (git cannot add it)") - else: - added = True - if added: - subprocess.run( - ["git", "commit", "-q", "-m", "add invalid-utf8 name"], - cwd=tmp_root_path, - check=True, - capture_output=True, - ) - saved_root = ROOT - ROOT = tmp_root_path - try: - hub_tracked.cache_clear() - tracked = hub_tracked(rev="HEAD") - finally: - ROOT = saved_root - hub_tracked.cache_clear() - got = bad_name in tracked - if not got: - ok = False - print( - f" {'ok ' if got else 'FAIL'} want=True got={got!s:<24} hub_tracked: a non-UTF-8 filename round-trips instead of crashing" - ) - - # Region extraction and hashing: a forked github-release block must hash differently from the canonical. - region = split_jobs(rel_ok).get("github-release") - forked_region = split_jobs( - rel_ok.replace(" merge-multiple: true\n", " artifact-ids: 1\n") - ).get("github-release") - if ( - region is None - or forked_region is None - or content_hash(region) == content_hash(forked_region) - ): - ok = False - print(" FAIL verbatim: forked github-release region should hash differently") - else: - print( - " ok verbatim: a forked github-release region hashes differently from the canonical" - ) - # Section-region extraction, where the region includes the heading line, keeps a nested ### and a fenced ## inside the body, ends at the next sibling H2, is None where absent, and rehashes where the heading is re-cased. - # The per-section verbatim check depends on every one of these. - md = "# Title\n\n## Alpha\n\nbody a\n\n```\n## not a heading\n```\n\n### nested\nstill alpha\n\n## Beta\n\nbody b\n" - a, b, gone = ( - extract_section(md, "Alpha"), - extract_section(md, "Beta"), - extract_section(md, "Gamma"), - ) - spaced = extract_section( - "## Alpha\n\nbody a\n", "Alpha" - ) # extra marker-gap whitespace still locates - if ( - a is None - or not a.startswith("## Alpha") - or "body a" not in a - or "## not a heading" not in a - or "still alpha" not in a - or "body b" in a - or b is None - or not b.startswith("## Beta") - or "body b" not in b - or "body a" in b - or gone is not None - or spaced is None - or not spaced.startswith("## Alpha") - or content_hash(a) - == content_hash(extract_section(md.replace("## Alpha", "## alpha"), "Alpha")) - ): - ok = False - print(" FAIL section: extract_section region/hash behavior") - else: - print( - " ok section: heading in region, fenced ## kept, sibling H2 ends, None if absent, whitespace-tolerant locate, re-cased heading rehashes" - ) - # A mismatched marker nested inside a fence (a ~~~ line inside a ``` block) must not end the fence early. - # _fence_step carries this contract, and extract_section and strip_sections each consume it. - nested = "## Alpha\n```md\n~~~\n## Fake\n```\nreal content\n" - nested_extract = extract_section(nested, "Alpha") - nested_strip = strip_sections( - "## Keep\ntext\n```md\n~~~\n## Fake Drop\n```\nmore text\n## Drop\nreal drop content\n", - ["Drop"], - ) - if ( - nested_extract is None - or "## Fake" not in nested_extract - or "real content" not in nested_extract - or "## Fake Drop" not in nested_strip - or "real drop content" in nested_strip - or "more text" not in nested_strip - ): - ok = False - print(" FAIL section: a mismatched marker nested in a fence ends the region early") - else: - print(" ok section: a mismatched marker nested in a fence is not a boundary") - - # Coordination-reference scan: the hub name inside a verbatim section is exempt, outside one is not. - # The first case is the real AGENTS.md shape, where the byte-locked Fleet Bootstrap block must name the hub and a repo therefore cannot clear a finding against it. - # The CRLF case matters because extract_section normalizes EOLs while a carried file can still arrive CRLF (an operational repo's Windows-native override, or a stale copy mid-resync), so excision must survive that. - boot = "## Fleet Bootstrap\n\nThe canonical rules live in `github.com/acme/Hub`.\n" - owned = "## Where the Rules Live\n\nReport a rule discrepancy to acme/Hub.\n" - clean_doc = ( - "# AGENTS\n\n" - + boot - + "\n## Where the Rules Live\n\nState the behavior, not the destination.\n" - ) - dirty_doc = "# AGENTS\n\n" + boot + "\n" + owned - tref = [ - ("hub name only inside the verbatim section", clean_doc, {"Fleet Bootstrap"}, False), - ("hub name in prose the repo owns", dirty_doc, {"Fleet Bootstrap"}, True), - ("same document with nothing declared verbatim still flags", clean_doc, set(), True), - ( - "CRLF document excises the same way", - clean_doc.replace("\n", "\r\n"), - {"Fleet Bootstrap"}, - False, - ), - ( - "a re-cased verbatim heading still excises", - clean_doc.replace("## Fleet Bootstrap", "## fleet bootstrap"), - {"Fleet Bootstrap"}, - False, - ), - ( - "no hub reference at all", - "# AGENTS\n\n## Where the Rules Live\n\nNothing to see.\n", - {"Fleet Bootstrap"}, - False, - ), - # A second region under the same heading is excised by name, which is right: both are that section. - ( - "the heading appearing twice excises both", - "# AGENTS\n\n" + boot + "\n## Notes\n\n" + boot, - {"Fleet Bootstrap"}, - False, - ), - # A fenced copy is not a heading, so it is prose the repo owns and the reference in it must flag. - # This is the case that made positional excision necessary, because removing the extracted text instead would delete the fenced copy along with the real region and the scan would fail open. - # That is the one arrangement a repo could otherwise use to carry the reference in a document it owns. - ( - "a fenced copy of the section is prose, not the section", - "# AGENTS\n\n## Notes\n\n```\n" + boot + "```\n\n" + boot, - {"Fleet Bootstrap"}, - True, - ), - ] - tref_ok = True - for label, doc, verb, want in tref: - got = template_ref_outside_verbatim(doc, verb, "acme/Hub") - if got != want: - ok = tref_ok = False - print(f" FAIL template-ref: {label} (expected {want}, got {got})") - if tref_ok: - print( - f" ok template-ref: {len(tref)} cases, verbatim regions excised before the hub-name scan" - ) - - # A version literal or commit SHA in an instruction document, outside its verbatim sections. - sha = "0123456789abcdef0123456789abcdef01234567" - vl = [ - ("a release number is flagged", "Pinned at hub release `2.0.657`.\n", set(), ["2.0.657"]), - ("a full SHA is flagged", f"uses: o/r/x.yml@{sha} # 2.0.1\n", set(), ["2.0.1", sha]), - ("an uppercase SHA is flagged", f"Pinned at {sha.upper()}.\n", set(), [sha.upper()]), - ("an abbreviated SHA is flagged", "Pinned at `f3b4cc9`.\n", set(), ["f3b4cc9"]), - ("an uppercase abbreviated SHA is flagged", "Pinned at `F3B4CC9`.\n", set(), ["F3B4CC9"]), - ("an all-letter hex word is not a SHA", "A facade over deadbeef.\n", set(), []), - ("a hex color is not a SHA", "Color #1f2937ff.\n", set(), []), - ("a hash-prefixed hex token is not a SHA", "See #f3b4cc9.\n", set(), []), - ("an identifier suffix is not a SHA", "Name foo_1a2b3c4d and 123e4567-e89b.\n", set(), []), - ("an image tag SHA is flagged", "Image app:sha-1a2b3c4.\n", set(), ["1a2b3c4"]), - ("a hex constant is not a SHA", "Mask 0x7fffffff.\n", set(), []), - ( - "a worked example is flagged", - "1.0.12 publishes as 1.0.13.\n", - set(), - ["1.0.12", "1.0.13"], - ), - ("a placeholder example is clean", "`1.0.N` publishes as `1.0.(N+1)`.\n", set(), []), - ("the mechanism alone is clean", "SHA-pinned to a hub release.\n", set(), []), - ("a dotted quad is not a version", "Host 192.168.1.10 serves it.\n", set(), []), - ("a two-part version is not a pin", "Python 3.13 and 3.14.\n", set(), []), - ( - "a literal inside a verbatim section is the hub's", - "# W\n\n## Locked\n\nPinned at 2.0.1.\n\n## Mine\n\nClean.\n", - {"Locked"}, - [], - ), - ( - "a literal outside the verbatim section is flagged", - "# W\n\n## Locked\n\nClean.\n\n## Mine\n\nPinned at 2.0.2.\n", - {"Locked"}, - ["2.0.2"], - ), - ] - vl_ok = True - for label, doc, verb, want in vl: - got = version_literals_outside_verbatim(doc, verb) - if got != sorted(want): - ok = vl_ok = False - print(f" FAIL version-literal: {label} (expected {sorted(want)}, got {got})") - if vl_ok: - print(f" ok version-literal: {len(vl)} cases, pins outside verbatim sections flagged") - - # An H2 the manifest does not declare, in AGENTS.md, GOVERNANCE.md, or .github/copilot-instructions.md. - # Fence-aware, so a heading syntax example or a shell comment inside a code sample is not misread as a real section. - uh = [ - ( - "a declared H2 is not flagged", - "# AGENTS\n\n## Fleet Bootstrap\n\nText.\n", - {"fleet bootstrap"}, - [], - ), - ( - "an undeclared H2 is flagged", - "# AGENTS\n\n## Fleet Bootstrap\n\nText.\n\n## Local Notes\n\nRepo-specific.\n", - {"fleet bootstrap"}, - ["local notes"], - ), - ( - "declared-name match is case-insensitive", - "# AGENTS\n\n## fleet BOOTSTRAP\n\nText.\n", - {"fleet bootstrap"}, - [], - ), - ( - "an un-normalized declared set (mixed case, untrimmed) is normalized here, not trusted", - "# AGENTS\n\n## Fleet Bootstrap\n\nText.\n", - {" Fleet Bootstrap "}, - [], - ), - ( - "an H1 title and a nested H3 are not judged, only H2", - "# Local Notes\n\n## Fleet Bootstrap\n\n### Local Notes\n\nText.\n", - {"fleet bootstrap"}, - [], - ), - ( - "a fenced sample showing heading syntax is not a real heading", - "# AGENTS\n\n## Fleet Bootstrap\n\n```\n## Local Notes\n```\n", - {"fleet bootstrap"}, - [], - ), - ( - "a backtick in a backtick-fence info string is not a valid opener, so the heading after it is real", - "# AGENTS\n\n## Fleet Bootstrap\n\n```md`\n## Local Notes\n", - {"fleet bootstrap"}, - ["local notes"], - ), - ( - "a repo's own local content, undeclared, is flagged even in a file with its own declared sections", - ( - "# Copilot Instructions\n\n## GitHub Copilot Review Runbook\n\nText.\n\n" - "## Development Workflow\n\nLocal build and test steps.\n\n" - "## Command Line Usage\n\nLocal CLI reference.\n" - ), - {"github copilot review runbook"}, - ["command line usage", "development workflow"], - ), - ] - uh_ok = True - for label, doc, declared, want in uh: - got = undeclared_h2_headings(doc, declared) - if got != want: - ok = uh_ok = False - print(f" FAIL undeclared-heading: {label} (expected {want}, got {got})") - if uh_ok: - print(f" ok undeclared-heading: {len(uh)} cases, H2-only, case-insensitive, fence-aware") - - # A fence closes only on a same-family marker at least as long as the opener. - uf = [ - ("a simple ``` fence excludes its content", "```\n## Phantom\n```\n## Real\n", "## Real\n"), - ("a simple ~~~ fence excludes its content", "~~~\n## Phantom\n~~~\n## Real\n", "## Real\n"), - ( - "a ~~~ line nested inside a ``` fence does not close it", - "```md\n~~~\n## Phantom\n```\n## Real\n", - "## Real\n", - ), - ( - "a shorter ``` cannot close a longer ```` fence", - "````md\n## Phantom\n```\n## Real\n````\n", - "", - ), - ( - "a longer closing fence than the opener still closes", - "```\n## Phantom\n````\n## Real\n", - "## Real\n", - ), - ( - "a marker run followed by trailing text does not close", - "```\n## Phantom\n```not-a-fence\n## Real\n```\n## Real2\n", - "## Real2\n", - ), - ( - "trailing whitespace after the marker run still closes", - "```\n## Phantom\n``` \n## Real\n", - "## Real\n", - ), - ( - "a 3-space-indented closing fence still closes", - "```\n## Phantom\n ```\n## Real\n", - "## Real\n", - ), - ( - "a 4-space-indented closing fence remains content, not a boundary", - "```\n## Phantom\n ```\n## Still Phantom\n```\n## Real\n", - "## Real\n", - ), - ( - "a 4-space-indented opener never opens a fence", - " ```\n## Real\n", - " ```\n## Real\n", - ), - ( - "a backtick in a backtick-fence info string is not a valid opener", - "```md`\n## Real\n", - "```md`\n## Real\n", - ), - ( - "a backtick in a tilde-fence info string does not disqualify it", - "~~~md`\n## Phantom\n~~~\n## Real\n", - "## Real\n", - ), - ] - uf_ok = True - for label, doc, want in uf: - got = unfenced_text(doc) - if got != want: - ok = uf_ok = False - print(f" FAIL unfenced-text: {label} (expected {want!r}, got {got!r})") - if uf_ok: - print( - f" ok unfenced-text: {len(uf)} cases, closing fence matches the opener's marker and length" - ) - - # Issue generator: findings land in the right buckets and the title carries the count. - fe = {"name": "Widget", "types": ["python"]} - it, ib = render_issue( - fe, - [("LETTER", "file: X absent"), ("DRIFT", "verbatim: Y differs"), ("ERROR", "gh failed")], - "main", - "abc1234", - "2026-01-01T00:00:00Z", - "hub123", - ) - et, eb = render_issue(fe, [], "main", "abc1234", "2026-01-01T00:00:00Z", "hub123") - if ( - "Widget" not in it - or "(3 findings)" not in it - or "## Must fix" not in ib - or "**LETTER** file: X absent" not in ib - or "## Converge" not in ib - or "verbatim: Y differs" not in ib - or "## Could not verify" not in ib - or "(0 findings)" not in et - or "nothing to converge" not in eb - ): - ok = False - print(" FAIL issue: render_issue grouping/title/empty behavior") - else: - print( - " ok issue: render_issue groups must-fix/converge/unverifiable, counts findings, handles the clean case" - ) - - # README/HISTORY mirror: same title+intro matches modulo the ToC-omit comment, and intro drift is caught. - r_md = "# Widget \n\nDoes widget things.\n\n## Build\n" - h_md = "# Widget\n\nDoes widget things.\n\n## Release History\n" - h_bad = "# Widget\n\nDoes other things.\n\n## Release History\n" - r_two = "# W\n\nLine one.\n\nLine two.\n\n## Build\n" - h_joined = "# W\n\nLine one.\nLine two.\n\n## Release History\n" - if ( - title_and_intro(r_md) != title_and_intro(h_md) - or title_and_intro(r_md) == title_and_intro(h_bad) - or title_and_intro(r_two) == title_and_intro(h_joined) - ): - ok = False - print(" FAIL history: README/HISTORY title+intro mirror detection") - else: - print( - " ok history: mirror matches modulo the ToC-omit comment, intro drift and paragraph-boundary drift detected" - ) - # Description mirror: links reduce to their text, and a link-free line passes through unchanged. - linked = "Utility to clean [media](https://x.example/Foo_(bar)) per the [spec][spec-ref]." - nested = "See [API [docs]](https://example.test/a_(b)_(c)) for details." - # A failed outer span used to jump past the whole run instead of retrying one character in, - # so a link nested inside a non-link bracket run was skipped. - inner_link = "See [[docs](url)] for details." - # A backslash-escaped `\[` used to count as real nesting, corrupting the label match instead - # of being read as a literal character. - escaped_bracket = r"See [API \[docs](url) for details." - if ( - strip_md_links(linked) != "Utility to clean media per the spec." - or strip_md_links("Plain intro line.") != "Plain intro line." - or strip_md_links(nested) != "See API [docs] for details." - or strip_md_links(inner_link) != "See [docs] for details." - or strip_md_links(escaped_bracket) != r"See API \[docs for details." - ): - ok = False - print(" FAIL description: strip_md_links behavior") - else: - print( - " ok description: Markdown links reduce to their text, plain text passes through, nested brackets/parens balance, nested and escaped labels handled" - ) - # A run of unmatched '[' used to re-scan the remaining text from every position, - # O(N^2) on a README tagline read before any length limit. Linear now: a slow run means a regression. - pathological = "[" * 20000 - start = time.monotonic() - strip_md_links(pathological) - elapsed = time.monotonic() - start - if elapsed > 1.0: - ok = False - print( - f" FAIL description: strip_md_links took {elapsed:.2f}s on unmatched brackets, expected linear" - ) - else: - print( - f" ok description: strip_md_links stays linear on unmatched brackets ({elapsed:.3f}s for 20000)" - ) - # cspell duplication: a workspace cSpell word list is detected, and a mere cspell.json mention is not. - ws_dup = '{ "settings": { "cSpell.words": ["foo"] } }' - ws_ok = '{ "settings": { "editor.rulers": [100] }, "note": "words live in cspell.json" }' - if not workspace_cspell_words(ws_dup) or workspace_cspell_words(ws_ok): - ok = False - print(" FAIL cspell: workspace word-list detection") - else: - print( - " ok cspell: workspace cSpell word list detected, a plain cspell.json mention is not" - ) - - # The branch-drift direction split, covering a modify, add and delete on main plus a develop-only change. - bd_base = {"keep": "a", "moda": "1", "modb": "2", "deld": "e", "devonly": "x"} - bd_main = { - "keep": "a", - "moda": "9", - "modb": "9", - "add": "n", - "devonly": "x", - } # moved moda/modb, added 'add', deleted 'deld' - bd_dev = { - "keep": "a", - "moda": "1", - "modb": "7", - "add": "m", - "deld": "e", - "devonly": "y", - } # still at base on moda/deld, moved modb/add/devonly - bd_behind, bd_diverged = classify_branch_drift(bd_base, bd_main, bd_dev) - if bd_behind != ["deld", "moda"] or bd_diverged != ["add", "modb"]: - ok = False - print(f" FAIL branch-drift classify -> behind={bd_behind} diverged={bd_diverged}") - else: - print( - " ok branch-drift: behind (modify/delete develop still at base) vs diverged (both moved), develop-only excluded" - ) - - with tempfile.TemporaryDirectory() as tmp_root: - tmp_path = pathlib.Path(tmp_root) - full = tmp_path / "full" - full.mkdir() - for cmd in ( - ["git", "init", "-q"], - ["git", "config", "user.email", "test@test.invalid"], - ["git", "config", "user.name", "test"], - ["git", "config", "commit.gpgsign", "false"], - ["git", "commit", "-q", "--allow-empty", "-m", "one"], - ["git", "commit", "-q", "--allow-empty", "-m", "two"], - ): - subprocess.run(cmd, cwd=full, check=True, capture_output=True) - shallow = tmp_path / "shallow" - subprocess.run( - ["git", "clone", "-q", "--depth", "1", full.as_uri(), str(shallow)], - check=True, - capture_output=True, - ) - saved_root = ROOT - try: - for label, root, want_refuse in (("shallow", shallow, True), ("full", full, False)): - ROOT = root - _require_full_history.cache_clear() - _git_revisions.cache_clear() - git_blob_in_file_history.cache_clear() - reads = ( - ("revisions", lambda: _git_revisions("absent.txt", rev="HEAD")), - ("blob", lambda: git_blob_in_file_history("absent.txt", "0" * 40, rev="HEAD")), - ) - for read, call in reads: - try: - call() - msg = None - except ShallowCloneError as e: - msg = str(e) - good = (msg is not None) == want_refuse - if want_refuse: - good = ( - good and "Run `git fetch --unshallow origin` in that checkout." in msg - ) - if not good: - ok = False - print( - f" {'ok ' if good else 'FAIL'} shallow-clone guard: {label} clone, {read}" - ) - finally: - ROOT = saved_root - _require_full_history.cache_clear() - _git_revisions.cache_clear() - git_blob_in_file_history.cache_clear() - - with tempfile.TemporaryDirectory() as tmp_root: - hub_dir = pathlib.Path(tmp_root) / "hub" - hub_dir.mkdir() - rels = ( - "registry/repos.json", - "repo-config/settings.json", - "spec/secrets.json", - "spec/files.json", - "spec/project-types.json", - "spec/readme-sections.json", - "spec/third-party-tools.json", - "spec/divergences.json", - ) - for rel in rels: - (hub_dir / rel).parent.mkdir(parents=True, exist_ok=True) - (hub_dir / rel).write_bytes((ROOT / rel).read_bytes()) - for cmd in ( - ["git", "init", "-q"], - ["git", "config", "user.email", "test@test.invalid"], - ["git", "config", "user.name", "test"], - ["git", "config", "commit.gpgsign", "false"], - ["git", "add", "."], - ["git", "commit", "-q", "-m", "one"], - ["git", "commit", "-q", "--allow-empty", "-m", "two"], - ): - subprocess.run(cmd, cwd=hub_dir, check=True, capture_output=True) - shallow_hub = pathlib.Path(tmp_root) / "shallow-hub" - subprocess.run( - ["git", "clone", "-q", "--depth", "1", hub_dir.as_uri(), str(shallow_hub)], - check=True, - capture_output=True, - ) - name = next( - r["name"] - for r in load("registry/repos.json")["repos"] - if r.get("status") == "cataloged" - ) - saved_root = ROOT - saved_audit_repo = audit_repo - - def history_read_only(entry, spec, branch=None): - _git_revisions("absent.txt", rev="HEAD") - return [], "" - - try: - ROOT = shallow_hub - globals()["audit_repo"] = history_read_only - for flag_argv, label, want_code in ( - ([name], "sweep", 1), - (["--issue", name], "issue", 2), - ): - _require_full_history.cache_clear() - _git_revisions.cache_clear() - out, err = io.StringIO(), io.StringIO() - try: - with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): - code = main(flag_argv) - except Exception as e: # noqa: BLE001 - code = f"raised {type(e).__name__}" - text = out.getvalue() if label == "sweep" else err.getvalue() - good = ( - code == want_code - and "Run `git fetch --unshallow origin` in that checkout." in text - and (label == "sweep" or not out.getvalue()) - ) - if label == "sweep": - good = good and "ERROR" in text - if not good: - ok = False - print( - f" {'ok ' if good else 'FAIL'} shallow-clone refusal, public {label} path (code {code})" - ) - finally: - ROOT = saved_root - globals()["audit_repo"] = saved_audit_repo - _require_full_history.cache_clear() - _git_revisions.cache_clear() - - # CLI parsing, where a repo name and a flag value must not be confused for one another. - # The previous hand-rolled parse took every non `--` argument as a repo name, so `--branch develop` would have audited a repo called "develop" rather than overriding the branch. - cli_cases = [ - ([], [], None, False, False), - (["Utilities"], ["Utilities"], None, False, False), - (["Utilities", "--branch", "develop"], ["Utilities"], "develop", False, False), - (["--branch=feature/x", "Utilities"], ["Utilities"], "feature/x", False, False), - (["--issue", "Utilities"], ["Utilities"], None, True, False), - (["--selftest"], [], None, False, True), - ] - for argv, names, branch, issue, selftest in cli_cases: - got = parse_args(argv) - if (got.names, got.branch, got.issue, got.selftest) != (names, branch, issue, selftest): - ok = False - print(f" FAIL cli parse {argv} -> {got}") - else: - print(f" ok cli: {argv or ['(no args)']} -> names={names} branch={branch}") - - # The override wins over the registry field, the field wins over the registry defaults, and the fleet value is main. - # The two defaults cases are what pin the middle step: before it existed every reader fell back to a literal "main", so a declared defaults.groundTruthBranch validated clean and was never read. - ground_cases = [ - ({}, None, None, "main"), - ({"groundTruthBranch": "develop"}, None, None, "develop"), - ({"groundTruthBranch": "main"}, "develop", None, "develop"), - ({}, "feature/x", None, "feature/x"), - ({}, None, {"groundTruthBranch": "develop"}, "develop"), - ({"groundTruthBranch": "main"}, None, {"groundTruthBranch": "develop"}, "main"), - # The two empty-string cases are what pin presence over truthiness at each level. - # spec/validate.py refuses a declared "" by name at both, so returning it is what lets that refusal be the thing the operator reads, where defaulting it away would address main and report nothing. - ({"groundTruthBranch": ""}, None, {"groundTruthBranch": "develop"}, ""), - ({}, None, {"groundTruthBranch": ""}, ""), - ] - for entry, branch, defaults, want in ground_cases: - got = ground_branch_of(entry, branch, defaults) - if got != want: - ok = False - print(f" FAIL ground branch {entry} + {branch} + {defaults} -> {got}, want {want}") - else: - print( - f" ok ground branch: registry={entry.get('groundTruthBranch')} defaults={(defaults or {}).get('groundTruthBranch')} override={branch} -> {want}" - ) - - # The driftNote freshness rules, driven directly rather than through audit_repo. - # That exercises both the clean and the unclean audit without standing up a whole conformant repo to reach one branch. - note_spec = { - "types": { - "types": { - "hugo": {"checks": [{"id": "hugo.generator.pinned"}]}, - "docker": {"checks": [{"id": "docker.cache.registry"}]}, - }, - "crossCutting": {"setup": {"checks": [{"id": "setup.driftnotes.current"}]}}, - } - } - note_cases = [ - # The end-of-sentence period is the case an end-anchored pattern misses, and it is how both notes this form was introduced for are written. - ( - "check id followed by a period", - ["hugo"], - ["Pinned in two workflows (hugo.generator.pinned)."], - 0, - ["does not evaluate"], - ), - ( - "check id mid-sentence", - ["hugo"], - ["The pin (hugo.generator.pinned) is duplicated."], - 0, - ["does not evaluate"], - ), - ( - "cross-cutting id, which no repo declares", - ["hugo"], - ["Notes go stale (setup.driftnotes.current)."], - 0, - ["does not evaluate"], - ), - ( - "check id of an undeclared type", - ["hugo"], - ["Cache layer (docker.cache.registry)."], - 0, - ["this repo does not declare"], - ), - ( - "a malformed types value does not abort the note check", - None, - ["Vendored helper, not tracked (python.directories.declared)."], - 0, - [], - ), - ( - "a detection suppression on a repo without the type", - ["hugo"], - ["Vendored helper, not tracked (python.directories.declared)."], - 0, - [], - ), - ( - "check id absent from the catalog", - ["hugo"], - ["Theme record (hugo.vendored.provenence)."], - 0, - ["does not define"], - ), - # A version string in parentheses is the false positive the id shape has to exclude. - ( - "parenthesized version, not a check id", - ["hugo"], - ["Generator held at (0.164.0) by the composite action."], - 0, - [], - ), - ( - "permanent deviation, no marker and no id", - ["hugo"], - ["Relies on validate-task, having no get-version-task."], - 0, - [], - ), - ( - "marker note on a clean audit", - ["hugo"], - ["The sibling doc is pending fleet ratification."], - 0, - ["but the audit is clean"], - ), - # The case the old gate suppressed outright: one unclearable finding, and the note never checked. - ( - "marker note with findings open", - ["hugo"], - ["The sibling doc is pending fleet ratification."], - 1, - ["while 1 finding(s) are open"], - ), - ] - for label, types, notes, open_count, wanted in note_cases: - got = driftnote_findings({"types": types, "driftNotes": notes}, note_spec, open_count) - texts = [t for _, t in got] - if len(texts) != len(wanted) or not all(w in t for w, t in zip(wanted, texts)): - ok = False - print(f" FAIL driftNote {label} -> {texts}, want {wanted}") - else: - print(f" ok driftNote {label}: {len(texts)} finding(s)") - - # A spec with no catalog and an entry with a check-id note fails loudly and names what is missing. - # No live caller pairs the two, since main() always loads the catalog. - # The pairing stays an error rather than becoming a fallback, because resolving ids against an absent catalog reports every one of them undefined. - # That is a work list which destroys correct notes. - try: - driftnote_findings( - {"types": ["hugo"], "driftNotes": ["A note (hugo.build.strict)."]}, {"registry": {}}, 0 - ) - ok = False - print(" FAIL missing catalog: no error raised") - except KeyError as e: - if "project-types.json" not in str(e): - ok = False - print(f" FAIL missing catalog: error does not name the file -> {e}") - else: - print(" ok missing catalog: raises and names spec/project-types.json") - - # The README structure engine, run against the real declared model rather than a fixture of it. - # A model edit that contradicts these checks then fails here instead of on the fleet. - rm = load("spec/readme-sections.json") - conformant = ( - "# Fixture\n\nA fixture repository.\n\n## Build and Distribution\n\n- **Source Code**: [GitHub][gh]\n\n" - "### Build Status\n\n[![Release Status][a]][x]\\\n[![Last Commit][b]][x]\n\n" - "### Releases\n\n[![GitHub Release][c]][x]\\\n[![GitHub Pre-Release][d]][x]\n\n" - "### Release Notes\n\n**Version**: 1.0\n\n## Table of Contents\n\n- [Overview](#overview)\n\n" - "## Overview\n\nWhat it does.\n\n## Whatever This Repo Calls It\n\nRepo-specific.\n\n" - "## Questions or Issues\n\nOpen an issue.\n\n## 3rd Party Tools\n\n- [Thing][t]\n\n" - "## License\n\nLicensed under the [MIT License][license]\\\n![License][license-shield]\n\n" - "\n\n" - "[a]: https://img.shields.io/github/actions/workflow/status/o/r/publish-release.yml?label=Releases%20Build\n" - "[b]: https://img.shields.io/github/last-commit/o/r\n" - "[c]: https://img.shields.io/github/v/release/o/r?label=GitHub%20Release\n" - "[d]: https://img.shields.io/github/v/release/o/r?include_prereleases&filter=*-g*&label=GitHub%20Pre-Release\n" - "[license-shield]: https://img.shields.io/github/license/o/r\n" - ) - # The four repos that suffix every heading for the ToC extension must read identically to the plain form. - omit_toc = re.sub( - r"^(#{2,3} .*)$", r"\1 ", conformant, flags=re.MULTILINE - ) - readme_cases = [ - ("a conformant README, repo-specific section included", conformant, set(), True, 0), - ( - "a second intro paragraph is not a finding", - conformant.replace( - "A fixture repository.\n", - "A fixture repository.\n\nAnd a clarifying paragraph about it.\n", - ), - set(), - True, - 0, - ), - ( - "a retired section name reports a rename", - conformant.replace("## Overview", "## Features"), - set(), - True, - 1, - ), - ( - "Questions or Issues is optional while private", - conformant.replace("## Questions or Issues\n\nOpen an issue.\n\n", ""), - set(), - False, - 0, - ), - ( - "Questions or Issues is required when public", - conformant.replace("## Questions or Issues\n\nOpen an issue.\n\n", ""), - set(), - True, - 1, - ), - ( - "3rd Party Tools is required in every repo", - conformant.replace("## 3rd Party Tools\n\n- [Thing][t]\n\n", ""), - set(), - True, - 1, - ), - ( - "a Table of Contents is required with no size threshold", - conformant.replace("## Table of Contents\n\n- [Overview](#overview)\n\n", ""), - set(), - True, - 1, - ), - ( - "an out-of-order declared section reports once", - conformant.replace("## Questions or Issues\n\nOpen an issue.\n\n", "").replace( - "## Overview", "## Questions or Issues\n\nOpen an issue.\n\n## Overview" - ), - set(), - True, - 1, - ), - ( - "License must be the last section", - conformant + "\n## TODO\n\nA backlog.\n", - set(), - True, - 1, - ), - ( - "a missing Release Notes sub-section reports once", - conformant.replace("### Release Notes\n\n**Version**: 1.0\n\n", ""), - set(), - True, - 1, - ), - ( - "a `## ` line inside a fence is not a section", - conformant.replace("What it does.", "```md\n## Not A Section\n```"), - set(), - True, - 0, - ), - ("Usage and Installation are N/A for source-only", conformant, {"source-only"}, True, 0), - ("a ToC-omit comment on every heading changes nothing", omit_toc, set(), True, 0), - ] - for label, text, sel_types, public, wantn in readme_cases: - got = readme_section_findings(text, rm, sel_types, public) - if len(got) != wantn: - ok = False - print( - f" {'ok ' if len(got) == wantn else 'FAIL'} want={wantn} got={len(got)} readme sections: {label}" - ) - if len(got) != wantn: - for _, t in got: - print(f" {t}") - - docker_shield = "[![Docker Latest][e]][x]\n[e]: https://img.shields.io/docker/v/o/r/latest\n" - inline_all = re.sub( - r"!\[([^\]]*)\]\[([a-z-]+)\]", - lambda m: f"![{m.group(1)}](https://img.shields.io/PLACEHOLDER-{m.group(2)})", - conformant, - ) - shield_cases = [ - ("base shields only, a repo publishing nothing", conformant, {}, 0), - ("a ToC-omit comment on every heading changes nothing", omit_toc, {}, 0), - ("a docker repo owes a version shield", conformant, {"publish": [{"target": "docker"}]}, 1), - ( - "a docker repo carrying one is satisfied, whatever its channel", - conformant.replace( - "[![GitHub Pre-Release][d]][x]\n", - "[![GitHub Pre-Release][d]][x]\\\n" + docker_shield, - ), - {"publish": [{"target": "docker"}]}, - 0, - ), - ( - "a nuget repo owes one version shield, no prerelease", - conformant, - {"publish": [{"target": "nuget"}]}, - 1, - ), - ( - "a pypi repo owes one version shield, no prerelease", - conformant, - {"publish": [{"target": "pypi"}]}, - 1, - ), - ( - "a caption the repo spells differently is not a finding", - conformant.replace("![Release Status]", "![Lint Build]"), - {}, - 0, - ), - ( - "an extra shield beyond the class is not a finding", - conformant.replace( - "[![Last Commit][b]][x]", "[![Last Commit][b]][x]\\\n[![Last Build][a]][x]" - ), - {}, - 0, - ), - ( - "a shield in the wrong sub-section does not count as present", - conformant.replace("[![GitHub Release][c]][x]\\\n", ""), - {}, - 1, - ), - # A badge shown as a code sample is markup, so it satisfies nothing and trips nothing. - # Every region the README checks read comes through readme_region, which strips fences at the root. - ( - "a fenced badge sample does not satisfy a required shield", - conformant.replace( - "[![GitHub Release][c]][x]\\\n", "```md\n[![GitHub Release][c]][x]\n```\n" - ), - {}, - 1, - ), - ( - "a fenced license shield does not trip the exclusive rule", - conformant.replace( - "## Overview", "```md\n![License][license-shield]\n```\n\n## Overview" - ), - {}, - 0, - ), - ( - "a retired badge service is reported wherever it sits", - conformant.replace( - "[license-shield]: https://img.shields.io/github/license/o/r\n", - "[license-shield]: https://img.shields.io/github/license/o/r\n[last-build-shield]: https://byob.yarr.is/o/r/lastbuild\n", - ), - {}, - 1, - ), - # The case above named every placement and read only the reference block, so an inline badge was invisible rather than wrong. - # That is the reading shield_endpoints already takes for every other shield, and this one had not taken it. - ( - "a retired badge written inline is reported", - conformant.replace( - "[![Last Commit][b]][x]", - "[![Last Commit][b]][x]\\\n![Last Build](https://byob.yarr.is/o/r/lastbuild)", - ), - {}, - 1, - ), - ( - "a retired badge defined and rendered is one finding, not two", - conformant.replace( - "[![Last Commit][b]][x]", - "[![Last Commit][b]][x]\\\n![Last Build][last-build-shield]", - ).replace( - "[license-shield]: https://img.shields.io/github/license/o/r\n", - "[license-shield]: https://img.shields.io/github/license/o/r\n[last-build-shield]: https://byob.yarr.is/o/r/lastbuild\n", - ), - {}, - 1, - ), - ( - "a retired badge shown as a fenced sample is markup", - conformant.replace( - "## Overview", - "```md\n![Last Build](https://byob.yarr.is/o/r/lastbuild)\n```\n\n## Overview", - ), - {}, - 0, - ), - # The wording follows which of the three shapes it is, since a definition nothing renders is not rendering anything. - # Saying it renders sends the reader looking for a badge that is not on the page. - ( - "an unrendered definition says so rather than claiming a render", - conformant.replace( - "[license-shield]: https://img.shields.io/github/license/o/r\n", - "[license-shield]: https://img.shields.io/github/license/o/r\n[last-build-shield]: https://byob.yarr.is/o/r/lastbuild\n", - ), - {}, - 1, - "nothing renders it", - ), - ( - "a rendered definition says renders", - conformant.replace( - "[![Last Commit][b]][x]", - "[![Last Commit][b]][x]\\\n![Last Build][last-build-shield]", - ).replace( - "[license-shield]: https://img.shields.io/github/license/o/r\n", - "[license-shield]: https://img.shields.io/github/license/o/r\n[last-build-shield]: https://byob.yarr.is/o/r/lastbuild\n", - ), - {}, - 1, - "renders the byob", - ), - # The same endpoint rendered inline leaves this definition unused, so attributing by URL would credit the render to a reference nothing uses. - ( - "an unused definition beside an inline render is not credited with it", - conformant.replace( - "[![Last Commit][b]][x]", - "[![Last Commit][b]][x]\\\n![Last Build](https://byob.yarr.is/o/r/lastbuild)", - ).replace( - "[license-shield]: https://img.shields.io/github/license/o/r\n", - "[license-shield]: https://img.shields.io/github/license/o/r\n[last-build-shield]: https://byob.yarr.is/o/r/lastbuild\n", - ), - {}, - 1, - "rendered elsewhere", - ), - ( - "the pre-release shield is told from the release shield by its query", - conformant.replace( - "?include_prereleases&filter=*-g*&label=GitHub%20Pre-Release", - "?label=Another%20Release", - ), - {}, - 1, - ), - ( - "a pre-release shield without the prerelease filter is a finding", - conformant.replace("&filter=*-g*&", "&"), - {}, - 1, - "with `filter=*-g*`", - ), - ( - "a missing release shield names the query it forbids", - conformant.replace("[![GitHub Release][c]][x]\\\n", ""), - {}, - 1, - "without `include_prereleases`", - ), - # The license shield is an ordinary member of the base class, addressed to a different section. - ("the license shield in the closing License section", conformant, {}, 0), - ( - "no license shield at all", - conformant.replace("\\\n![License][license-shield]", ""), - {}, - 1, - ), - ( - "the license shield left under Releases", - conformant.replace("\\\n![License][license-shield]", "").replace( - "[![GitHub Pre-Release][d]][x]", - "[![GitHub Pre-Release][d]][x]\\\n![License][license-shield]", - ), - {}, - 2, - ), - ( - "a shield's own reference definition is not a use of it", - conformant.replace("![License][license-shield]\n", ""), - {}, - 1, - ), - # An inline shield is resolved rather than silently invisible. - # Reading references alone let a repo writing every badge inline pass by carrying nothing the check could see. - ( - "inline shields count as present", - inline_all.replace("PLACEHOLDER-a", "github/actions/workflow/status/o/r") - .replace("PLACEHOLDER-b", "github/last-commit/o/r") - .replace("PLACEHOLDER-c", "github/v/release/o/r") - .replace("PLACEHOLDER-d", "github/v/release/o/r?include_prereleases&filter=*-g*") - .replace("PLACEHOLDER-license-shield", "github/license/o/r"), - {}, - 0, - ), - ( - "an inline shield in the wrong section is still exclusive", - conformant.replace( - "## Overview", - "![License](https://img.shields.io/github/license/o/r)\n\n## Overview", - ), - {}, - 1, - ), - ] - # A case may carry a fifth element, a substring the finding text must contain. - # A count alone cannot tell one wording from another, and the wording is the whole subject of some of these cases. - for case in shield_cases: - label, text, ent, wantn = case[:4] - want_text = case[4] if len(case) > 4 else None - got = readme_shield_findings(text, rm, ent) - good = len(got) == wantn and (want_text is None or any(want_text in t for _, t in got)) - if not good: - ok = False - shown = f"want={wantn}" if want_text is None else f"want={wantn}+'{want_text}'" - print(f" {'ok ' if good else 'FAIL'} {shown} got={len(got)} readme shields: {label}") - if not good: - for _, t in got: - print(f" {t}") - - links_ok = ( - "# F\n\nA fixture.\n\n## X\n\n[a](#x) [b][gh] [c][docker-hub-link] [d][agents] [e][upstream-link]\n\n" - "![License][license-shield]\n\n" - "\n\n[x-anchor]: #x\n\n\n\n[license-shield]: https://img.shields.io/github/license/o/r\n\n" - "\n\n[actions-link]: https://github.com/o/r/actions\n[docker-hub-link]: https://hub.docker.com/r/o/r\n[github-link]: https://github.com/o/r\n\n" - "\n\n[agents]: ./AGENTS.md\n[license]: ./LICENSE\n\n" - "\n\n[upstream-link]: https://github.com/someone-else/their-repo\n" - ) - # Kept in reference-name order, so the case measures the per-target rule rather than tripping the sort check. - two_images = links_ok.replace( - "[github-link]: https://github.com/o/r\n", - "[github-link]: https://github.com/o/r\n[lsio-docker-hub-link]: https://hub.docker.com/r/o/r-lsio\n", - ) - swapped = links_ok.replace( - "\n\n[x-anchor]: #x\n\n\n\n[license-shield]: https://img.shields.io/github/license/o/r\n", - "\n\n[license-shield]: https://img.shields.io/github/license/o/r\n\n\n\n[x-anchor]: #x\n", - ) - link_cases = [ - ("a conformant reference block", links_ok, 0, 0), - ( - "a shield reference not ending -shield", - links_ok.replace("[license-shield]", "[licence]"), - 1, - 0, - ), - ( - "a URI reference not ending -link", - links_ok.replace("[upstream-link]", "[upstream]"), - 1, - 0, - ), - ( - "a repo-local reference ending -link", - links_ok.replace("[agents]:", "[agents-link]:"), - 1, - 0, - ), - ( - "the repo root named for the project", - links_ok.replace("[github-link]", "[fixture-link]"), - 1, - 0, - ), - ("somebody else's GitHub repo is not renamed", links_ok, 0, 0), - ("one Docker Hub image takes the bare name", links_ok, 0, 0), - ("two Docker Hub images each take a target prefix", two_images, 1, 0), - ( - "two Docker Hub images, both prefixed, is clean", - two_images.replace( - "[docker-hub-link]: https://hub.docker.com/r/o/r\n", - "[base-docker-hub-link]: https://hub.docker.com/r/o/r\n", - ), - 0, - 0, - ), - ( - "an undeclared group header", - links_ok.replace("", ""), - 0, - 1, - ), - ("groups out of the declared order", swapped, 0, 1), - ( - "a group not sorted by reference name", - links_ok.replace( - "[agents]: ./AGENTS.md\n[license]: ./LICENSE\n", - "[license]: ./LICENSE\n[agents]: ./AGENTS.md\n", - ), - 0, - 1, - ), - ( - "a reference in the wrong group", - links_ok.replace("[license]: ./LICENSE\n", "").replace( - "\n", "\n\n[license]: ./LICENSE\n" - ), - 0, - 1, - ), - # A shields.io URL the document never renders is judged by that usage rather than by its host, so it is an ordinary URI. - # Across the fleet every one of the 119 img.shields.io definitions is rendered, so a host test decides nothing. - ( - "an unrendered shields.io reference is a URI, not a shield", - links_ok.replace( - "[license-shield]: https://img.shields.io/github/license/o/r\n", - "[extra-shield]: https://img.shields.io/badge/never-rendered-blue\n[license-shield]: https://img.shields.io/github/license/o/r\n", - ), - 1, - 1, - ), - # A code sample showing badge markup is markup rather than a badge. - # Reading a fenced block would invent a group header, two definitions, and a rendered shield the document does not have. - ( - "a fenced code sample is not read as definitions", - links_ok - + "\n## Sample\n\n```md\n\n\n[wrong-name]: https://example.test/\n![License][license-shield]\n```\n", - 0, - 0, - ), - # A dependency hosted where this project also publishes stays external, so it keeps its own name. - # The owner-scoped prefixes are what separate the two, and they live in the model rather than in the code. - ( - "somebody else's NuGet package is not renamed nuget-link", - links_ok.replace( - "[upstream-link]: https://github.com/someone-else/their-repo", - "[upstream-link]: https://www.nuget.org/packages/Serilog/", - ), - 0, - 0, - ), - ( - "this project's own NuGet package is renamed", - links_ok.replace( - "[upstream-link]: https://github.com/someone-else/their-repo", - "[upstream-link]: https://www.nuget.org/packages/o.Widget/", - ), - 1, - 1, - ), - # A comment that opens no group is not a group header. - # A markdownlint directive sits on its own line exactly like one, and reading it as a group reports a header the document never declared. - ( - "a directive comment is not a link group", - "\n" + links_ok, - 0, - 0, - ), - # Also when it sits inside the reference block, where definitions do fall under it. - ( - "a directive inside the reference block is not a link group", - links_ok.replace( - "[agents]: ./AGENTS.md", - "\n[agents]: ./AGENTS.md", - ), - 0, - 0, - ), - # A scheme makes a target a link rather than a file, so it is named `-link` and grouped External. - # Reading `mailto:` as a repo path would demand a bare name and the Repo group for a contact address. - ( - "a mailto target is a URI, not a repo path", - links_ok.replace( - "[upstream-link]: https://github.com/someone-else/their-repo", - "[upstream-link]: mailto:someone@example.test", - ), - 0, - 0, - ), - ( - "a mailto target named as a repo path is reported", - links_ok.replace( - "[upstream-link]: https://github.com/someone-else/their-repo", - "[upstream]: mailto:someone@example.test", - ), - 1, - 0, - ), - # Two names may point at one URL, and only the rendered one is a shield. - # Keying on the URL instead would make the plain link a shield because its twin is rendered. - ( - "a second reference to a rendered URL is not itself a shield", - links_ok.replace( - "[upstream-link]: https://github.com/someone-else/their-repo", - "[upstream-link]: https://img.shields.io/github/license/o/r", - ), - 0, - 0, - ), - ] - for label, text, want_letter, want_drift in link_cases: - got = readme_link_findings(text, rm, "o/r") - gl = sum(1 for k, _ in got if k == "LETTER") - gd = sum(1 for k, _ in got if k == "DRIFT") - good = (gl, gd) == (want_letter, want_drift) - if not good: - ok = False - print( - f" {'ok ' if good else 'FAIL'} want={want_letter}L/{want_drift}D got={gl}L/{gd}D readme links: {label}" - ) - if not good: - for k, t in got: - print(f" {k}: {t}") - - # The shared tool catalog, checked over the intersection only: a repo's own tools are its own business. - cat = load("spec/third-party-tools.json") - tools_head = "# F\n\nA fixture.\n\n## 3rd Party Tools\n\n" - tools_defs = "\n[cspell-link]: https://cspell.org\n[widget-link]: https://widget.example/\n" - tool_cases = [ - ( - "a catalog tool matching the catalog", - tools_head - + "| Tool | Role |\n| --- | --- |\n| [cspell][cspell-link] | Spell checker. |\n" - + tools_defs, - 0, - ), - ( - "a catalog tool described differently", - tools_head - + "| Tool | Role |\n| --- | --- |\n| [cspell][cspell-link] | Spell-checks README.md in CI. |\n" - + tools_defs, - 1, - ), - ( - "a catalog tool with no description at all", - tools_head + "- [cspell][cspell-link]\n" + tools_defs, - 1, - ), - ( - "a catalog tool linked by another URL", - tools_head - + "| Tool | Role |\n| --- | --- |\n| [cspell][widget-link] | Spell checker. |\n" - + tools_defs, - 1, - ), - ( - "a tool the catalog does not name is not judged", - tools_head - + "| Tool | Role |\n| --- | --- |\n| [Widget][widget-link] | whatever this repo calls it |\n" - + tools_defs, - 0, - ), - ( - "the bullet form is read like the table form", - tools_head + "- [cspell][cspell-link] - Spell checker.\n" + tools_defs, - 0, - ), - ( - "no section yields nothing, since its absence is already reported", - "# F\n\nA fixture.\n\n## Other\n\nx\n", - 0, - ), - # A definition inside a code sample is markup being shown, so it must not resolve a tool link. - # The sample sits after the real definition deliberately, so reading across fences would override it. - ( - "a fenced sample does not supply a tool link", - tools_head - + "| Tool | Role |\n| --- | --- |\n| [cspell][cspell-link] | Spell checker. |\n" - + tools_defs - + "\n```md\n[cspell-link]: https://wrong.example/\n```\n", - 0, - ), - # A tool table shown as a sample inside the section is markup, not entries. - ( - "a fenced tool table is not read as entries", - tools_head - + "| Tool | Role |\n| --- | --- |\n| [cspell][cspell-link] | Spell checker. |\n\n```md\n| [cspell][cspell-link] | wrong description |\n```\n" - + tools_defs, - 0, - ), - # The list is alphabetized, which spec/readme-structure.md states and nothing checked until now. - ( - "an unsorted tool list is reported", - tools_head - + "| Tool | Role |\n| --- | --- |\n| [markdownlint-cli2][md-link] | Markdown linter. |\n| [cspell][cspell-link] | Spell checker. |\n" - + tools_defs - + "[md-link]: https://github.com/DavidAnson/markdownlint-cli2\n", - 1, - ), - ( - "a sorted tool list is not", - tools_head - + "| Tool | Role |\n| --- | --- |\n| [cspell][cspell-link] | Spell checker. |\n| [markdownlint-cli2][md-link] | Markdown linter. |\n" - + tools_defs - + "[md-link]: https://github.com/DavidAnson/markdownlint-cli2\n", - 0, - ), - # GitHub's Markdown makes both outer pipes optional, so a row written without either is an entry rather than a non-row. - # Requiring them read such a table as empty, which scores as clean rather than as unread. - ( - "a row with no trailing pipe is read", - tools_head - + "| Tool | Role |\n| --- | --- |\n| [cspell][cspell-link] | Spell checker.\n" - + tools_defs, - 0, - ), - ( - "a row with no trailing pipe is judged like any other", - tools_head - + "| Tool | Role |\n| --- | --- |\n| [cspell][cspell-link] | Spell-checks the README in CI.\n" - + tools_defs, - 1, - ), - ( - "a whole table with no trailing pipes is ordered", - tools_head - + "| Tool | Role\n| --- | ---\n| [markdownlint-cli2][md-link] | Markdown linter.\n| [cspell][cspell-link] | Spell checker.\n" - + tools_defs - + "[md-link]: https://github.com/DavidAnson/markdownlint-cli2\n", - 1, - ), - ( - "a row with no leading pipe is read", - tools_head - + "Tool | Role |\n--- | --- |\n[cspell][cspell-link] | Spell checker. |\n" - + tools_defs, - 0, - ), - ( - "a table with neither outer pipe is judged", - tools_head - + "Tool | Role\n--- | ---\n[cspell][cspell-link] | Spell-checks the README in CI.\n" - + tools_defs, - 1, - ), - # The License column is forbidden by spec/readme-structure.md, and project-types.json asserted that as `letter` before anything read it. - # One finding is raised for the table, read off its header, rather than one per row. - ( - "a License column is reported once", - tools_head - + "| Tool | Role | License |\n| --- | --- | --- |\n| [cspell][cspell-link] | Spell checker. | MIT |\n| [markdownlint-cli2][md-link] | Markdown linter. | MIT |\n" - + tools_defs - + "[md-link]: https://github.com/DavidAnson/markdownlint-cli2\n", - 1, - ), - ( - "a License column does not stop the rows being read", - tools_head - + "| Tool | Role | License |\n| --- | --- | --- |\n| [cspell][cspell-link] | Spell-checks the README in CI. | MIT |\n" - + tools_defs, - 2, - ), - ( - "a table with no License column is not reported", - tools_head - + "| Tool | Role |\n| --- | --- |\n| [cspell][cspell-link] | Spell checker. |\n" - + tools_defs, - 0, - ), - # A tool whose own role is the word license must not read as the column, since the header is a header. - ( - "a row cell reading License is not the column", - tools_head - + "| Tool | Role |\n| --- | --- |\n| [Widget][widget-link] | License |\n" - + tools_defs, - 0, - ), - ( - "a License column with no outer pipes is reported", - tools_head - + "Tool | Role | License\n--- | --- | ---\n[cspell][cspell-link] | Spell checker. | MIT\n" - + tools_defs, - 1, - ), - # A thematic break carries no pipe, so it is not a one-cell delimiter row and the paragraph above it is not a header. - ( - "a thematic break is not a delimiter row", - tools_head - + "Licensing notes follow.\n\n---\n\n| Tool | Role |\n| --- | --- |\n| [cspell][cspell-link] | Spell checker. |\n" - + tools_defs, - 0, - ), - # Prose mentioning a tool is not a row, since a link alone does not make one. - ( - "prose naming a cataloged tool is not a row", - tools_head + "We run [cspell][cspell-link] in CI.\n" + tools_defs, - 0, - ), - ] - for label, text, wantn in tool_cases: - got = third_party_tool_findings(text, cat) - if len(got) != wantn: - ok = False - print( - f" {'ok ' if len(got) == wantn else 'FAIL'} want={wantn} got={len(got)} 3rd party tools: {label}" - ) - if len(got) != wantn: - for _, t in got: - print(f" {t}") - - # A canonicalLinks entry that can match nothing is a name nothing enforces, so it raises rather than skipping. - # Skipping would quietly stop checking that destination, which is the failure mode this whole dimension exists to avoid. - broken = dict(rm, canonicalLinks=list(rm["canonicalLinks"]) + [{"name": "orphan-link"}]) - try: - canonical_link_entry("https://example.test/", broken, "o/r") - ok = False - print(" FAIL canonicalLinks entry with neither repoPath nor match: no error raised") - except KeyError as e: - if "readme-sections.json" not in str(e) or "orphan-link" not in str(e): - ok = False - print(f" FAIL canonicalLinks entry error does not name the file and entry -> {e}") - else: - print( - " ok canonicalLinks entry with neither repoPath nor match: raises, naming the file and the entry" - ) - - # The tagline is the first line of the intro region, so a README carrying a second paragraph still mirrors. - two_para = title_and_intro("# X\n\nThe tagline.\n\nA clarifying paragraph.\n\n## Next\n")[1] - if tagline(two_para) != "The tagline." or "\n" in tagline(two_para): - ok = False - print(f" FAIL tagline extraction -> {tagline(two_para)!r}") - else: - print(" ok tagline: the first line of the intro region, further paragraphs excluded") - - # description_findings(): the README/About/Docker Hub mirror set, with and without a declared registry field. - desc_readme = {"README.md": "# Fixture\n\nA short tagline.\n"} - desc_cases = [ - ( - "no declared field, About matches the README tagline", - desc_readme, - {}, - {"description": "A short tagline."}, - 0, - ), - ( - "no declared field, About mismatches the README tagline", - desc_readme, - {}, - {"description": "Something else."}, - 1, - ), - ( - "declared field matches both README and About", - desc_readme, - {"description": "A short tagline."}, - {"description": "A short tagline."}, - 0, - ), - ( - "declared field present, README tagline diverges from it", - desc_readme, - {"description": "The declared description."}, - {"description": "The declared description."}, - 1, - ), - ( - "declared field present, About diverges from it", - desc_readme, - {"description": "A short tagline."}, - {"description": "Something else."}, - 1, - ), - ( - "declared field present, no README at all, About diverges from it", - {}, - {"description": "The declared description."}, - {"description": "Something else."}, - 1, - ), - ( - "declared field present, no README at all, About already matches it", - {}, - {"description": "The declared description."}, - {"description": "The declared description."}, - 0, - ), - ( - "no declared field and no README, nothing to measure against", - {}, - {}, - {"description": "Anything."}, - 0, - ), - ( - "a non-string declared field is reported rather than crashing", - desc_readme, - {"description": 42}, - {"description": "A short tagline."}, - 1, - ), - ( - "an explicit null declared field is reported rather than read as absent", - desc_readme, - {"description": None}, - {"description": "A short tagline."}, - 1, - ), - ( - "a whitespace-only declared field is reported rather than read as absent", - desc_readme, - {"description": " "}, - {"description": "A short tagline."}, - 1, - ), - ( - "a padded declared field is a DEFECT rather than silently trimmed", - desc_readme, - {"description": " A short tagline. "}, - {"description": "A short tagline."}, - 1, - ), - ( - "a declared field with an embedded newline is a DEFECT rather than silently accepted", - desc_readme, - {"description": "A short tagline.\nA second line."}, - {"description": "A short tagline."}, - 1, - ), - ( - "a declared field carrying a Markdown link is a DEFECT rather than silently canonical", - desc_readme, - {"description": "See [docs](https://example.test) for more."}, - {"description": "A short tagline."}, - 1, - ), - ( - "a declared field over the 100-char cap is a DEFECT rather than silently canonical", - desc_readme, - {"description": "a" * 101}, - {"description": "A short tagline."}, - 1, - ), - ] - for label, doc_texts_fx, entry_fx, live_fx, wantn in desc_cases: - got = description_findings(doc_texts_fx, entry_fx, live_fx, "owner/Fixture") - if len(got) != wantn: - ok = False - print( - f" {'ok ' if len(got) == wantn else 'FAIL'} want={wantn} got={len(got)} description: {label}" - ) - if len(got) != wantn: - for _, t in got: - print(f" {t}") - # A null declared field is a DEFECT via validate.py's own contract, not a silent LETTER-only "About mismatch". - null_declared = description_findings( - desc_readme, {"description": None}, {"description": "A short tagline."}, "owner/Fixture" - ) - if not any( - k == "DEFECT" and "description must be a non-empty string" in t for k, t in null_declared - ): - ok = False - print(f" FAIL description: null-declared-field DEFECT contract -> {null_declared}") - else: - print(" ok description: a null declared field is a DEFECT via validate.py's contract") - # The DEFECT names the actual repo, not a generic "registry" label, so it stays actionable in a fleet-wide run. - if not any(k == "DEFECT" and t.startswith("owner/Fixture:") for k, t in null_declared): - ok = False - print(f" FAIL description: DEFECT does not name the repo -> {null_declared}") - else: - print(" ok description: a declared-field DEFECT names the repo, not a generic label") - # The declared field, once present, is what the wording names as the source - not "the README". - declared_mismatch = description_findings( - desc_readme, - {"description": "The declared description."}, - {"description": "Something else."}, - "owner/Fixture", - ) - if not any( - "declared description" in t and "registry/repos.json" in t for _, t in declared_mismatch - ): - ok = False - print(f" FAIL description: declared-field wording -> {declared_mismatch}") - else: - print( - " ok description: a declared field names registry/repos.json as the source, not the README" - ) - # A docker-publishing repo's Docker Hub short description is checked against the same canonical value. - real_dhd = globals()["docker_hub_description"] - globals()["docker_hub_description"] = lambda slug: "A stale Docker Hub blurb." - try: - docker_mismatch = description_findings( - desc_readme, - {"description": "A short tagline.", "publish": [{"target": "docker"}]}, - {"description": "A short tagline."}, - "owner/Fixture", - ) - finally: - globals()["docker_hub_description"] = real_dhd - if not any("Docker Hub short description" in t for _, t in docker_mismatch): - ok = False - print(f" FAIL description: Docker Hub mismatch not reported -> {docker_mismatch}") - else: - print(" ok description: a docker repo's stale Docker Hub short description is reported") - - # A ground-truth branch that does not resolve is one error, not a baseline's worth of letters. - # Every `?ref=` read would 404 and report each carried file absent, describing the ref, not the repo. - # The branch facts are already read at that point, so they are reported rather than dropped. - entry = {"name": "Fixture", "url": "https://github.com/owner/Fixture", "hasDevelop": True} - real_gh = globals()["gh"] - globals()["gh"] = lambda path, ok404=False: None if "/branches/" in path else {"private": False} - try: - missing_findings, missing_sha = audit_repo(entry, {"registry": {}}, "no-such-branch") - finally: - globals()["gh"] = real_gh - kinds = [k for k, _ in missing_findings] - errors = [t for k, t in missing_findings if k == "ERROR"] - if ( - missing_sha != "" - or len(errors) != 1 - or "no-such-branch" not in errors[0] - or kinds != ["DEFECT", "DRIFT", "ERROR"] - ): - ok = False - print(f" FAIL missing ground branch -> {missing_findings}") - else: - print( - " ok missing ground branch: the branch facts, then one error naming the ref, no letters" - ) - - # The hub-only set is the manifest subtracted from the hub's tracked files, so a declared path must never appear in it. - # Asserted against the live manifest rather than a fixture, since the failure this guards is a declared path leaking into the deletion list, which only the real pairing can show. - declared = {e["path"] for e in load("spec/files.json")["baseline"]} - hub_only = hub_only_paths({"files": load("spec/files.json")}, rev="HEAD") - leaked = sorted(declared & hub_only) - if leaked or not hub_only: - ok = False - print( - f" FAIL hub-only set -> {len(hub_only)} paths, {len(leaked)} declared leaked: {leaked}" - ) - else: - print( - f" ok hub-only set excludes every declared path ({len(hub_only)} hub-hosted paths, {len(declared)} declared)" - ) - - # A malformed ledger row is dropped rather than raising, so a hand-edit cannot take the audit down with it. - # The well-formed rows around it still resolve, which is what keeps a typo from silently clearing every disposition. - led = { - "divergences": { - "gaps": [ - {"path": "a/b.sh", "disposition": "retire", "reason": "r"}, - {"path": "c/d.md", "disposition": "accepted"}, - {"path": "no-disposition.md"}, - {"disposition": "retire"}, - "not-an-object", - ] - } - } - got_disp = gap_dispositions(led) - if got_disp != {"a/b.sh": ("retire", "r"), "c/d.md": ("accepted", "")}: - ok = False - print(f" FAIL gap disposition parsing -> {got_disp}") - else: - print( - " ok gap dispositions: well-formed rows parse, a row missing path or disposition is dropped" - ) - - # A truncated or unreadable tree returns None so the caller reports it, rather than a partial set that reads as a repo carrying nothing. - # A partial tree cannot tell a file the repo does not have from one the response stopped short of, and the second is the silent-narrowing case. - real_gh = globals()["gh"] - head = {"commit": {"commit": {"tree": {"sha": "deadbeef"}}}} - try: - globals()["gh"] = lambda path, ok404=False: { - "tree": [{"path": "x", "type": "blob"}], - "truncated": True, - } - truncated = repo_tree("o/r", head) - globals()["gh"] = lambda path, ok404=False: { - "tree": [{"path": "x", "type": "blob"}, {"path": "d", "type": "tree"}], - "truncated": False, - } - whole = repo_tree("o/r", head) - finally: - globals()["gh"] = real_gh - if truncated is not None or repo_tree("o/r", {"commit": {}}) is not None or whole != {"x"}: - ok = False - print(f" FAIL repo_tree -> truncated={truncated}, whole={whole}") - else: - print( - " ok repo_tree: a truncated tree and a missing tree sha both return None, and a whole one drops non-blobs" - ) - - real_gh = globals()["gh"] - - def holding(*paths): - def fake(path, ok404=False): - if "/commits?path=" in path: - name = path.split("/commits?path=", 1)[1].split("&", 1)[0] - stamp = {"commit": {"committer": {"date": "2000-01-01T00:00:00+00:00"}}} - return [stamp] if name in paths else [] - name = path.split("/contents/", 1)[1].split("?", 1)[0] - return ( - {"encoding": "base64", "content": "eA==", "name": name} if name in paths else None - ) - - return fake - - alt_cases = [ - (("codecov.yml",), ("codecov.yml", "codecov.yml"), "only the declared path"), - ((".codecov.yml",), (".codecov.yml", ".codecov.yml"), "only the alternative"), - (("codecov.yml", ".codecov.yml"), ("codecov.yml", "codecov.yml"), "both, declared wins"), - ((), ("codecov.yml", None), "neither"), - ] - try: - for held, (want_path, want_name), label in alt_cases: - globals()["gh"] = holding(*held) - got_path, got = read_baseline_file("o/r", "main", "codecov.yml", [".codecov.yml"]) - got_name = got["name"] if got is not None else None - if (got_path, got_name) != (want_path, want_name): - ok = False - print(f" FAIL read_baseline_file, {label} -> {got_path}, {got_name}") - else: - print( - f" ok read_baseline_file, {label}: read at {got_path}, found={got is not None}" - ) - real_hub_change = globals()["hub_last_change"] - real_canon_text = globals()["canonical_current_text"] - globals()["hub_last_change"] = lambda rel: ("2020-01-01T00:00:00+00:00", "abc1234") - globals()["canonical_current_text"] = lambda rel: "canonical" - unit = {"path": "codecov.yml", "fidelity": "intent", "intentRef": "WORKFLOW.md"} - try: - globals()["gh"] = holding(".codecov.yml") - alt_content, _, alt_findings = baseline_file_findings( - "o/r", "main", "codecov.yml", [".codecov.yml"], unit, False - ) - globals()["gh"] = holding() - _, _, none_findings = baseline_file_findings( - "o/r", "main", "codecov.yml", [".codecov.yml"], unit, False - ) - finally: - globals()["hub_last_change"] = real_hub_change - globals()["canonical_current_text"] = real_canon_text - stale_at_alt = [t for lvl, t in alt_findings if lvl == "DRIFT" and ".codecov.yml" in t] - if alt_content is None or len(alt_findings) != 1 or len(stale_at_alt) != 1: - ok = False - print(f" FAIL baseline_file_findings, only the alternative -> {alt_findings}") - else: - print( - " ok baseline_file_findings, only the alternative: present, its staleness read at the alternative" - ) - want_none = [ - ( - "LETTER", - "file: codecov.yml or .codecov.yml absent on main (verify intent per AUDIT.md section 7)", - ) - ] - if none_findings != want_none: - ok = False - print(f" FAIL baseline_file_findings, neither -> {none_findings}") - else: - print(" ok baseline_file_findings, neither: one LETTER naming both paths") - finally: - globals()["gh"] = real_gh - live_alts = declared_alternatives(load("spec/files.json")["baseline"]) - if live_alts.get("codecov.yml") != [".codecov.yml"]: - ok = False - print(f" FAIL declared_alternatives on the live manifest -> {live_alts}") - else: - print(" ok declared_alternatives: the live manifest names .codecov.yml for codecov.yml") - - # Applicability turns on tests, so a package-only build repo is N/A rather than owing a token for a report nothing produces. - type_mechs = {"csharp": "codecov", "python": "codecov"} - coverage_cases = [ - (["python"], {"src/pkg/__init__.py", "pyproject.toml"}, [], "python, no tests"), - (["python"], {"tests/test_a.py", "pyproject.toml"}, ["python"], "python with tests"), - ( - ["python"], - {"tests/test_a.py"}, - ["python"], - "python lint-only, with tests, owes the claim same as build", - ), - (["csharp"], {"src/App/App.csproj"}, [], "csharp, no test project"), - (["csharp"], {"test/App.Tests/App.Tests.csproj"}, ["csharp"], "csharp with tests"), - (["csharp"], {"src/Tests/App.csproj"}, ["csharp"], "csharp, located under Tests/"), - ( - ["csharp"], - {"test/Specs.csproj"}, - ["csharp"], - "csharp, located as a test, named nothing", - ), - ( - ["csharp"], - {"src/App/App.csproj", "docs/x.md"}, - [], - "csharp, neither named nor located as a test", - ), - ( - ["python"], - {"test_app.py", "pyproject.toml"}, - ["python"], - "python, root-level test module", - ), - ( - ["python"], - {"app_test.py", "pyproject.toml"}, - ["python"], - "python, trailing _test module", - ), - ( - ["python"], - {"src/app.py", "contest.py"}, - [], - "python, a module merely containing test", - ), - ( - ["python"], - {"tests/README.md", "pyproject.toml"}, - [], - "python, a test directory holding no Python", - ), - ( - ["python"], - {"tests/conftest.py", "pyproject.toml"}, - ["python"], - "python, a test directory holding Python", - ), - ( - ["csharp"], - {"test/App.UnitTest.csproj"}, - ["csharp"], - "csharp, singular Test in the name", - ), - ( - ["csharp"], - {"src/widget.tests.csproj"}, - ["csharp"], - "csharp, a lowercase test segment outside a test directory", - ), - ( - ["csharp"], - {"src/Contest.csproj", "src/Latest.csproj"}, - [], - "csharp, names carrying those letters inside a word rather than as a segment", - ), - ( - ["python"], - {"test/test_a.py", "pyproject.toml"}, - ["python"], - "python, singular test dir", - ), - (["python"], {"src/tests/test_a.py"}, ["python"], "python, tests below the root"), - (["python"], {"tests"}, [], "python, a file named tests is not a directory"), - ( - ["csharp", "python"], - {"test/App.Tests/App.Tests.csproj", "tools/py/pyproject.toml"}, - ["csharp"], - "mixed repo, the tested C# side claims coverage, the untested lint-only python side does not", - ), - (["python"], None, ["python"], "unreadable tree keeps the claim"), - (["docker"], {"tests/test_a.py"}, [], "a type mapped to no mechanism claims nothing"), - ] - coverage_ok = True - candidate_cases = [ - ( - ["csharp", "python"], - {"csharp": "lint-only", "python": "lint-only"}, - ["python"], - "only Python's lint-only owes coverage", - ), - (["csharp"], {"csharp": "build"}, ["csharp"], "a build language stays a candidate"), - (["csharp"], None, ["csharp"], "a malformed profiles value is read as none"), - ] - for types_in, profiles_in, expected, label in candidate_cases: - got = coverage_candidate_types(types_in, profiles_in) - if got != expected: - ok = False - coverage_ok = False - print(f" FAIL coverage_candidate_types [{label}] -> {got}, expected {expected}") - for types_in, tree_in, expected, label in coverage_cases: - got = coverage_claiming_types(types_in, type_mechs, tree_in) - if got != expected: - ok = False - coverage_ok = False - print(f" FAIL coverage_claiming_types [{label}] -> {got}, expected {expected}") - if coverage_ok: - print( - " ok coverage_claiming_types: tests decide the claim regardless of profile, and an unreadable tree keeps it" - ) - - # Constructed repo shapes throughout, never a registry entry, per GOVERNANCE.md "Representative Data - # in Agent-Authored Text": "Widget"/"Tools"/"Extra" are illustrative fixtures, not real repos. - python_type_cases = [ - ( - {"types": ["docs"]}, - {"tools/script.py"}, - 1, - "untyped python: a tracked .py file, no 'python' type", - ), - ({"types": ["python"]}, {"tools/script.py"}, 0, "typed python: 'python' already declared"), - ({"types": []}, None, 0, "unreadable tree: presence is unanswerable"), - ({"types": []}, {"README.md"}, 0, "no .py file anywhere: nothing to flag"), - ({"types": None}, {"tools/script.py"}, 1, "a malformed types value reads as untyped"), - ( - {"types": ["docs"]}, - {".husky/hub_fetch_run.py"}, - 0, - "the carried hook helper is not the repo's Python", - ), - ( - {"types": ["docs"]}, - {".husky/hub-fetch-run.py"}, - 0, - "the carried hook helper under its former name is not the repo's Python", - ), - ( - { - "types": ["docs"], - "driftNotes": ["Vendored tool, not tracked (python.directories.declared)."], - }, - {"tools/script.py"}, - 0, - "a driftNote naming the check suppresses the advisory", - ), - ( - {"types": ["docs"], "driftNotes": ["Retired helper (python.directories.declared)."]}, - {"README.md"}, - 1, - "a suppression with no .py file left is stale", - ), - ( - {"types": ["docs"], "driftNotes": ["python.directories.declared without parentheses"]}, - {"tools/script.py"}, - 1, - "only the parenthesized id suppresses", - ), - ] - python_type_ok = True - for entry_in, tree_in, expected, label in python_type_cases: - got = len(python_type_findings(entry_in, tree_in)) - if got != expected: - ok = False - python_type_ok = False - print(f" FAIL python_type_findings [{label}] -> {got} finding(s), expected {expected}") - if python_type_ok: - print( - " ok python_type_findings: a tracked .py file with no 'python' type is a suppressible advisory, and a stale suppression is reported" - ) - - plain = "100644" - exe = "100755" - file_naming_cases = [ - ( - "every conforming shape passes", - { - "scripts/fetch_report.py": plain, - "pkg/__init__.py": plain, - "tests/conftest.py": plain, - "scripts/sync-tools.sh": exe, - "scripts/sync-tools.ps1": plain, - "scripts/sync-tools.cmd": plain, - "scripts/sync-tools.bat": plain, - "tests/sync-tools.Tests.ps1": plain, - ".husky/pre-commit": exe, - "bin/run-backup": exe, - "PSScriptAnalyzerSettings.psd1": plain, - "Makefile": plain, - "README.md": plain, - ".envrc": exe, - }, - {}, - set(), - set(), - ), - ( - "a hyphenated or capitalized .py breaks", - {"scripts/fetch-report.py": plain, "scripts/FetchReport.py": plain, "tool.PY": plain}, - {}, - {"scripts/fetch-report.py", "scripts/FetchReport.py", "tool.PY"}, - set(), - ), - ( - "the carried hook helper under its former name breaks, beside the current name or alone", - {".husky/hub-fetch-run.py": plain, ".husky/hub_fetch_run.py": plain}, - {}, - {".husky/hub-fetch-run.py"}, - set(), - ), - ( - "a snake_case or capitalized shell, PowerShell, or batch name with no wrapped tool breaks", - { - "scripts/sync_tools.sh": exe, - "scripts/Sync-Tools.ps1": plain, - "scripts/SYNC.CMD": plain, - "scripts/sync-tools.SH": exe, - "scripts/sync_tools.bat": plain, - }, - {}, - { - "scripts/sync_tools.sh", - "scripts/Sync-Tools.ps1", - "scripts/SYNC.CMD", - "scripts/sync-tools.SH", - "scripts/sync_tools.bat", - }, - set(), - ), - ( - "a Pester suffix over a non-conforming stem breaks", - {"tests/Sync_Tools.Tests.ps1": plain}, - {}, - {"tests/Sync_Tools.Tests.ps1"}, - set(), - ), - ( - "a launcher beside its Python tool keeps the tool's stem", - { - "scripts/fetch_report.py": plain, - "scripts/fetch_report.sh": exe, - "scripts/fetch_report.ps1": plain, - "scripts/fetch_report.cmd": plain, - }, - {}, - set(), - {}, - ), - ( - "a launcher whose Python tool sits in another directory is undecided", - {"tools/fetch_report.py": plain, "scripts/fetch_report.sh": exe}, - {}, - set(), - {"scripts/fetch_report.sh"}, - ), - ( - "an extensionless launcher beside its Python tool may be an installed command, with no read", - {"scripts/fetch_report.py": plain, "scripts/fetch_report": exe}, - {}, - set(), - {"scripts/fetch_report"}, - ), - ( - "an extensionless launcher whose Python tool sits elsewhere is undecided, with no read", - {"tools/fetch_report.py": plain, "bin/fetch_report": exe}, - {}, - set(), - {"bin/fetch_report"}, - ), - ( - "an extensionless executable with a shebang and a non-conforming name breaks", - {"bin/run_backup": exe, "bin/RunBackup": exe}, - {"bin/run_backup": "#!/usr/bin/env bash\n", "bin/RunBackup": "#!/bin/sh\n"}, - {"bin/run_backup", "bin/RunBackup"}, - set(), - ), - ( - "an extensionless executable with no shebang, or no readable text, is undecided", - {"bin/Tool_Binary": exe, "bin/Unreadable_Tool": exe}, - {"bin/Tool_Binary": "\x7fELF", "bin/Unreadable_Tool": None}, - set(), - {"bin/Tool_Binary", "bin/Unreadable_Tool"}, - ), - ( - "an extensionless file without the executable bit is out of scope", - {"docs/Release_Notes": plain}, - {}, - set(), - set(), - ), - ( - "an executable dotfile and a dotted name are not judged, with no read", - {"bin/.Hidden_Hook": exe, "bin/Run_Me.v2": exe}, - {}, - set(), - set(), - ), - ( - "a dotfile with a listed extension is judged on its stem", - {"bin/.Hook_Run.sh": exe, "bin/.lint.py": plain}, - {}, - {"bin/.Hook_Run.sh", "bin/.lint.py"}, - set(), - ), - ] - file_naming_ok = True - for label, blobs_in, texts_in, want_broken, want_undecided in file_naming_cases: - reads = [] - - def read_text(path, texts_in=texts_in, reads=reads): - reads.append(path) - return texts_in.get(path) - - got = file_naming_findings({}, blobs_in, read_text) - broken_text = next((t for _, t in got if " break " in t), "") - undecided_text = next((t for _, t in got if "cannot decide" in t), "") - good = ( - all(kind == "DRIFT" for kind, _ in got) - and len(got) == bool(want_broken) + bool(want_undecided) - and broken_text.startswith( - f"file-naming: {len(want_broken)} file(s)" if want_broken else "" - ) - and all(f"{p}," in f"{broken_text}," or f"{p} " in broken_text for p in want_broken) - and undecided_text.startswith( - f"file-naming: {len(want_undecided)} file(s)" if want_undecided else "" - ) - and all(f"{p} (" in undecided_text for p in want_undecided) - and set(reads) == set(texts_in) - ) - if not good: - ok = False - file_naming_ok = False - print(f" FAIL file_naming_findings [{label}] -> {got}, reads {reads}") - many = {f"scripts/Tool{n}.py": plain for n in range(7)} - capped = file_naming_findings({}, many, lambda path: None) - if not (len(capped) == 1 and capped[0][1].endswith(" and 2 more")): - ok = False - file_naming_ok = False - print(f" FAIL file_naming_findings [past five paths the list is capped] -> {capped}") - exempt_note = "A package format fixes `tools/Install.ps1` by that name (filenaming.scripts)." - exemption_blobs = { - "tools/Install.ps1": plain, - "tools/Setup_Host.sh": exe, - "bin/Tool_Binary": exe, - } - exemption_cases = [ - ( - "a named path is exempt and an unnamed break still reports", - [exempt_note], - exemption_blobs, - {"tools/Setup_Host.sh"}, - {"bin/Tool_Binary"}, - [], - ), - ( - "an undecided path can be exempted too", - ["An installed command fixed by its vendor, `bin/Tool_Binary` (filenaming.scripts)."], - exemption_blobs, - {"tools/Install.ps1", "tools/Setup_Host.sh"}, - set(), - [], - ), - ( - "a note naming a path that no longer breaks, or no longer exists, is stale", - [ - "Tool-fixed `tools/Install.ps1`, `tools/install-host.sh`, `tools/Gone.ps1` (filenaming.scripts)." - ], - {"tools/Install.ps1": plain, "tools/install-host.sh": exe}, - set(), - set(), - ["exempts 2 path(s)", "tools/install-host.sh, tools/Gone.ps1"], - ), - ( - "a note naming no path exempts nothing and is reported", - ["Install scripts are named by the package format (filenaming.scripts)."], - {"tools/Install.ps1": plain}, - {"tools/Install.ps1"}, - set(), - ["names no path"], - ), - ( - "a path without the parenthesized id exempts nothing", - ["A package format fixes `tools/Install.ps1` (filenaming.scripts is pending)."], - {"tools/Install.ps1": plain}, - {"tools/Install.ps1"}, - set(), - [], - ), - ] - for label, notes_in, blobs_in, want_broken, want_undecided, want_notes in exemption_cases: - got = file_naming_findings({"driftNotes": notes_in}, blobs_in, lambda path: "\x7fELF") - note_texts = [t for _, t in got if t.startswith("registry: ")] - broken_text = next((t for _, t in got if " break " in t), "") - undecided_text = next((t for _, t in got if "cannot decide" in t), "") - good = ( - all(kind == "DRIFT" for kind, _ in got) - and len(got) == len(note_texts) + bool(want_broken) + bool(want_undecided) - and len(note_texts) == (1 if want_notes else 0) - and all(w in note_texts[0] for w in want_notes) - and all(p in broken_text for p in want_broken) - and all(p not in broken_text for p in blobs_in.keys() - want_broken) - and all(p in undecided_text for p in want_undecided) - and all(p not in undecided_text for p in blobs_in.keys() - want_undecided) - ) - if not good: - ok = False - file_naming_ok = False - print(f" FAIL file_naming_findings [{label}] -> {got}") - if driftnote_findings({"types": ["docs"], "driftNotes": [exempt_note]}, note_spec, 0) != []: - ok = False - file_naming_ok = False - print(" FAIL file_naming_findings [an exemption note is not surfaced as a check to judge]") - if file_naming_findings({}, None, lambda path: None) != []: - ok = False - file_naming_ok = False - print(" FAIL file_naming_findings [an unreadable tree reports nothing here]") - if file_naming_ok: - print( - " ok file_naming_findings: each script extension takes its stem shape, the wrapper, Pester, and recorded per-path exemptions hold, a stale exemption is reported, and an undecidable file is reported for judgment" - ) - - secret_stem_cases = [ - ( - "conforming stems, examples, and the directory's own files pass", - { - ".secrets/README.md", - ".secrets/.gitignore", - ".secrets/db_root_password.example", - ".secrets/mail_relay.env.example", - "deploy/.secrets/service_account.json.example", - "docs/Mixed-Case.md", - }, - set(), - ), - ( - "a hyphenated, capitalized, or bare stem breaks", - { - ".secrets/db-root-password.example", - ".secrets/Mail_Relay.env.example", - ".secrets/.env.example", - "deploy/.secrets/api-token", - }, - { - ".secrets/db-root-password.example", - ".secrets/Mail_Relay.env.example", - ".secrets/.env.example", - "deploy/.secrets/api-token", - }, - ), - ( - "a file outside a .secrets directory, or nested below one, is out of scope", - { - "secrets/db-root-password.example", - "config/.secrets-notes/Odd-Name", - ".secrets/docs/Mixed-Case.md", - }, - set(), - ), - ] - secret_stem_ok = True - for label, tree_in, want in secret_stem_cases: - got = secret_stem_findings(tree_in) - good = (not want and got == []) or ( - len(got) == 1 - and got[0][0] == "DRIFT" - and got[0][1].startswith(f"runtime-secrets: {len(want)} secret file(s)") - and all(p in got[0][1] for p in want) - ) - if not good: - ok = False - secret_stem_ok = False - print(f" FAIL secret_stem_findings [{label}] -> {got}") - if secret_stem_findings(None) != []: - ok = False - secret_stem_ok = False - print(" FAIL secret_stem_findings [an unreadable tree reports nothing here]") - if secret_stem_ok: - print( - " ok secret_stem_findings: a .secrets/ file stem that is not lowercase snake_case is DRIFT, the name read up to its first dot" - ) - - declared_types_cases = [ - ({"types": ["python", "docs"]}, ["python", "docs"], "a list of strings is read whole"), - ({"types": ["python", 3, None]}, ["python"], "a mixed list keeps its string elements"), - ({"types": "python"}, [], "a string is not a list"), - ({"types": {"python": 1}}, [], "an object is not a list"), - ({"types": None}, [], "a declared null reads as untyped"), - ({}, [], "an absent field reads as untyped"), - ] - declared_types_ok = True - for entry_in, expected, label in declared_types_cases: - got = declared_types(entry_in) - if got != expected: - ok = False - declared_types_ok = False - print(f" FAIL declared_types [{label}] -> {got!r}, expected {expected!r}") - if declared_types_ok: - print( - " ok declared_types: a malformed types value reads as its string elements or as an empty list" - ) - - python_directory_cases = [ - ( - {"pythonDirectories": ["Tools"]}, - {"Tools/a.py", "Extra/b.py"}, - 1, - "declared directory: one file outside it is missed", - ), - ( - {"pythonDirectories": ["Tools"]}, - {"Tools/a.py", "Tools/sub/b.py"}, - 0, - "declared directory: a nested file is still covered", - ), - ( - {}, - {"tools/a.py"}, - 1, - "no declaration and no root pyproject.toml: nothing is expected to cover it", - ), - ( - {}, - {"pyproject.toml", "a.py", "sub/b.py"}, - 0, - "no declaration, root pyproject.toml: '.' fallback covers everything", - ), - ({"pythonDirectories": ["Tools"]}, None, 0, "unreadable tree: coverage is unanswerable"), - ( - {"pythonDirectories": ["Tools"]}, - {f"Extra/f{i}.py" for i in range(7)}, - 1, - "more than 5 missed: one finding, capped listing", - ), - ] - python_directory_ok = True - # Each case describes a Python-typed repo unless it names its own types, since an untyped one gets only the type advisory. - python_directory_cases.append( - ({"types": ["docs"]}, {"tools/x.py"}, 0, "an untyped repo gets only the type advisory") - ) - python_directory_cases.append( - ( - {"pythonDirectories": ["Tools"]}, - {"Tools/a.py", "hub_fetch_run.py"}, - 0, - "the carried hook helper sits outside every directory", - ) - ) - python_directory_cases += [ - ({"types": "python"}, {"tools/x.py"}, 0, "a string types value reads as untyped"), - ({"types": None}, {"tools/x.py"}, 0, "a null types value reads as untyped"), - ] - for entry_in, tree_in, expected, label in python_directory_cases: - got_findings = python_directory_coverage_findings( - {"types": ["python"], **entry_in}, tree_in - ) - if len(got_findings) != expected: - ok = False - python_directory_ok = False - print( - f" FAIL python_directory_coverage_findings [{label}] -> {len(got_findings)} finding(s), expected {expected}" - ) - elif expected and "more" not in got_findings[0][1] and len(tree_in or []) > 5: - ok = False - python_directory_ok = False - print( - f" FAIL python_directory_coverage_findings [{label}] -> missing the '... and N more' cap: {got_findings[0][1]}" - ) - if python_directory_ok: - print( - " ok python_directory_coverage_findings: a .py file outside every expected directory is DRIFT, capped at 5 shown" - ) - - py_caller_uses = " uses: ./.github/workflows/validate-task.yml\n" - py_caller_block = ( - "jobs:\n validate:\n" - + py_caller_uses - + " with:\n python-directories: |\n Tools\n" - ) - py_caller_plain = ( - "jobs:\n validate:\n" + py_caller_uses + " with:\n python-directories: Tools\n" - ) - py_caller_no_with = "jobs:\n validate:\n" + py_caller_uses - py_caller_no_hub = "jobs:\n validate:\n runs-on: ubuntu-latest\n steps:\n - run: echo inline lint\n" - py_caller_path = ".github/workflows/test-pull-request.yml" - python_caller_cases = [ - ( - py_caller_path, - py_caller_block, - {"pythonDirectories": ["Tools"]}, - 0, - "block scalar matches the registry", - ), - ( - py_caller_path, - py_caller_block, - {"pythonDirectories": ["Other"]}, - 1, - "block scalar disagrees with the registry", - ), - ( - py_caller_path, - py_caller_plain, - {"pythonDirectories": ["Tools"]}, - 0, - "plain single value matches the registry", - ), - ( - py_caller_path, - py_caller_no_with, - {}, - 0, - "no with: input, registry declares none - both empty", - ), - ( - py_caller_path, - py_caller_no_with, - {"pythonDirectories": ["Tools"]}, - 1, - "no with: input, registry declares one", - ), - ( - ".github/workflows/other.yml", - py_caller_block, - {"pythonDirectories": ["Other"]}, - 0, - "not a scoped caller path", - ), - ( - py_caller_path, - py_caller_no_hub, - {"pythonDirectories": ["Tools"]}, - 0, - "validate job never reaches validate-task.yml", - ), - ( - py_caller_path, - py_caller_block.replace(" validate:\n", " check:\n"), - {"pythonDirectories": ["Tools"]}, - 1, - "a renamed job is reported, not skipped", - ), - ( - py_caller_path, - "jobs:\n validate:\n" + py_caller_uses + " with: { python-directories: Tools }\n", - {}, - 1, - "a flow-mapping with: is reported, not misread as declaring nothing", - ), - ( - py_caller_path, - py_caller_block + " validate-extra:\n" + py_caller_uses, - {"pythonDirectories": ["Tools"]}, - 1, - "a second job calling validate-task.yml is reported beside a matching validate", - ), - ] - py_caller_root = py_caller_plain.replace("python-directories: Tools", "python-directories: .") - python_caller_cases.append( - (py_caller_path, py_caller_root, {}, 1, "naming the root is not the same as naming nothing") - ) - py_caller_folded = py_caller_block.replace( - "python-directories: |\n Tools\n", - "python-directories: >\n Tools\n Other\n", - ) - py_caller_commented = py_caller_plain.replace( - " with:\n", " with:\n # a note indented past the keys\n" - ) - python_caller_cases.append( - ( - py_caller_path, - py_caller_commented, - {"pythonDirectories": ["Tools"]}, - 0, - "a deeper comment hides no key", - ) - ) - py_caller_tabbed = py_caller_plain.replace( - "python-directories: Tools", "python-directories: Too\tls" - ) - py_caller_value = py_caller_plain.replace("python-directories: Tools", "python-directories: {}") - python_caller_cases += [ - ( - py_caller_path, - py_caller_value.format('&dirs "Tools\\nOther"'), - {}, - 1, - "an anchored escaped double-quoted value is refused", - ), - ( - py_caller_path, - py_caller_value.format('!!str "Tools\\nOther"'), - {}, - 1, - "a tagged escaped double-quoted value is refused", - ), - ( - py_caller_path, - py_caller_value.format('\n "Tools\\nOther"'), - {}, - 1, - "an escaped double-quoted value on the next line is refused", - ), - ( - py_caller_path, - py_caller_value.format('\n # a note at the key column\n "Tools\\nOther"'), - {}, - 1, - "a comment at the key column hides no next-line value", - ), - ( - py_caller_path, - py_caller_value.format('"Tools\n \\nOther"'), - {"pythonDirectories": ["Tools"]}, - 1, - "an escape on a double-quoted value's continuation line is refused", - ), - ( - py_caller_path, - py_caller_value.format('"Tools" # a\\b'), - {"pythonDirectories": ["Tools"]}, - 0, - "a backslash past the closing quote is not an escape", - ), - ( - py_caller_path, - py_caller_value.format("&dirs >\n Tools"), - {"pythonDirectories": ["Tools"]}, - 1, - "an anchored folded scalar is refused", - ), - ( - py_caller_path, - py_caller_folded, - {"pythonDirectories": ["Other", "Tools"]}, - 1, - "a folded scalar is refused", - ), - ( - py_caller_path, - py_caller_tabbed, - {"pythonDirectories": ["Too\tls"]}, - 1, - "a tabbed entry resolves to nothing", - ), - ] - undeclared_root_cases = [ - ( - {}, - {"pyproject.toml", "tests/test_a.py", "a.py"}, - 1, - "a lint-only root suite nothing runs", - ), - ({}, {"pyproject.toml", "tests/test_a.py", "uv.lock"}, 0, "a locked root runs by default"), - ( - {}, - {"pyproject.toml", "tests/test_a.py", "requirements.txt"}, - 0, - "a pip root runs by default", - ), - ( - {}, - {"pyproject.toml", "tests/test_a.py", "docs/requirements.txt"}, - 1, - "a nested manifest is not the root's", - ), - ( - {"pythonDirectories": ["."]}, - {"pyproject.toml", "tests/test_a.py"}, - 0, - "a declared root runs", - ), - ({}, {"pyproject.toml", "a.py"}, 0, "no root suite to run"), - ( - {}, - {"pyproject.toml", "tests/App.Tests/App.Tests.csproj"}, - 0, - "a tests/ holding no Python", - ), - ({}, None, 0, "unreadable tree reports nothing"), - ] - undeclared_root_cases.append( - ( - {"types": ["docs"]}, - {"pyproject.toml", "tests/test_a.py"}, - 0, - "an untyped repo gets only the type advisory", - ) - ) - undeclared_root_cases += [ - ( - {"types": "python"}, - {"pyproject.toml", "tests/test_a.py"}, - 0, - "a string types value reads as untyped", - ), - ( - {"types": None}, - {"pyproject.toml", "tests/test_a.py"}, - 0, - "a null types value reads as untyped", - ), - ] - for entry_in, tree_in, expected, label in undeclared_root_cases: - got = len(python_undeclared_root_findings({"types": ["python"], **entry_in}, tree_in)) - if got != expected: - ok = False - print(f" FAIL python_undeclared_root_findings [{label}] -> {got}, expected {expected}") - python_caller_ok = True - for path_in, text_in, entry_in, expected, label in python_caller_cases: - got = len(python_directories_caller_findings(path_in, text_in, entry_in)) - if got != expected: - ok = False - python_caller_ok = False - print( - f" FAIL python_directories_caller_findings [{label}] -> {got} finding(s), expected {expected}" - ) - py_caller_escaped = py_caller_plain.replace( - "python-directories: Tools", 'python-directories: "Tools\\nOther"' - ) - escaped_got = python_directories_caller_findings( - py_caller_path, py_caller_escaped, {"pythonDirectories": ["Other", "Tools"]} - ) - if len(escaped_got) != 1 or "backslash escape" not in escaped_got[0][1]: - ok = False - python_caller_ok = False - print( - f" FAIL python_directories_caller_findings [an escaped double-quoted value is refused] -> {escaped_got}" - ) - if python_caller_ok: - print( - " ok python_directories_caller_findings: a caller's declared python-directories input is compared against the registry" - ) - - concurrency_ok = True - snippet_dir = ROOT / "catalog" / "snippets" / "workflows" - pub_path = ".github/workflows/publish-release.yml" - bot_path = ".github/workflows/merge-bot-pull-request.yml" - pub_text = (snippet_dir / "publish-release.yml").read_text(encoding="utf-8") - bot_text = (snippet_dir / "merge-bot-pull-request.yml").read_text(encoding="utf-8") - pub_group = "group: ${{ github.workflow }}-${{ github.ref }}" - pub_line = next(ln for ln in pub_text.splitlines() if ln.strip().startswith("group:")) - bot_line = next(ln for ln in bot_text.splitlines() if ln.strip().startswith("group:")) - pub_value = pub_line.split("group:", 1)[1].strip() - sched_line = ( - " " - + pub_group - + "${{ github.event_name == 'schedule' && format('-{0}', github.run_id) || '' }}" - ) - pub_gate = ( - "if: ${{ needs.plan.outputs.publish == 'true' && needs.validate.result == 'success' }}" - ) - sched_gate = "if: ${{ github.event_name == 'workflow_dispatch' && needs.plan.outputs.publish == 'true' }}" - sched_text = re.sub( - r"(?s)\non:\n.*?\n(?=\S)", - "\non:\n workflow_dispatch:\n schedule:\n - cron: '0 2 * * MON'\n\n", - pub_text.replace(pub_line, sched_line).replace(pub_gate, sched_gate), - count=1, - ) - concurrency_cases = [ - ( - pub_path, - sched_text, - 0, - "a schedule-only suffix on a publisher whose schedule never publishes", - ), - ( - pub_path, - pub_text.replace(pub_line, sched_line), - 1, - "a schedule-only suffix on a push-triggered publisher", - ), - ( - pub_path, - sched_text.replace(sched_gate, pub_gate), - 1, - "a schedule-only suffix on a publisher whose schedule publishes", - ), - ( - pub_path, - sched_text.replace( - sched_gate, - "if: ${{ github.event_name == 'workflow_dispatch' || needs.plan.outputs.publish == 'true' }}", - ), - 1, - "a dispatch gate behind a disjunction", - ), - ( - pub_path, - sched_text.replace( - sched_gate, - "if: >-\n github.event_name == 'workflow_dispatch' &&\n needs.plan.outputs.publish == 'true'", - ), - 0, - "a folded dispatch gate", - ), - ( - pub_path, - sched_text.replace( - sched_gate, - "if: >- # note\n github.event_name == 'workflow_dispatch' &&\n needs.plan.outputs.publish == 'true'", - ), - 0, - "a folded dispatch gate with a comment on its key line", - ), - ( - pub_path, - sched_text.replace("\non:\n", "\non: # triggers\n", 1), - 0, - "a comment on the on: key line", - ), - ( - pub_path, - sched_text.replace(sched_gate, "if: ${{ github.event_name == 'workflow_dispatch' }}"), - 0, - "a dispatch test alone", - ), - ( - pub_path, - sched_text.replace( - sched_gate, - "if: ${{ needs.plan.outputs.publish == 'true' && (github.event_name != 'schedule') }}", - ), - 1, - "a parenthesized not-schedule test as the last conjunct", - ), - ( - pub_path, - sched_text.replace( - sched_gate, - "if: ${{ needs.plan.outputs.publish == 'true' && (github.event_name == 'workflow_dispatch') }}", - ), - 0, - "a parenthesized dispatch test as the last conjunct", - ), - ( - pub_path, - sched_text.replace( - sched_gate, - "if: ${{ !(needs.plan.outputs.publish == 'true' && (github.event_name == 'workflow_dispatch')) }}", - ), - 1, - "a dispatch test inside a negated group", - ), - ( - pub_path, - sched_text.replace("build-release-task.yml@", "other-task.yml@"), - 1, - "a schedule-only suffix with no release job", - ), - ( - pub_path, - sched_text.replace(" schedule:\n", " push:\n schedule:\n", 1), - 1, - "a schedule-only suffix beside a push trigger", - ), - (pub_path, pub_text, 0, "the reference publisher, run_id suffix included"), - (bot_path, bot_text, 0, "the reference merge-bot"), - ( - pub_path, - pub_text.replace( - "github.actor != 'ptr727-codegen[bot]' && github.actor != 'dependabot[bot]'", - "github.actor != 'dependabot[bot]' && github.actor != 'ptr727-codegen[bot]'", - ), - 0, - "the allowlisted actors in the other order", - ), - ( - pub_path, - pub_text.replace(pub_line, " " + pub_group), - 0, - "a bare ref-scoped publisher group", - ), - ( - pub_path, - pub_text.replace(pub_line, " group: ${{ github.workflow }}"), - 1, - "a global publisher group", - ), - (pub_path, pub_text.replace(pub_line, " group: publish"), 1, "a literal publisher group"), - ( - pub_path, - pub_text.replace(pub_line, " " + pub_group + "-extra"), - 1, - "a literal publisher suffix", - ), - ( - pub_path, - pub_text.replace(pub_line, " " + pub_group + "${{ github.sha }}"), - 1, - "a non-run_id publisher suffix", - ), - ( - pub_path, - pub_text.replace("cancel-in-progress: false", "cancel-in-progress: true"), - 1, - "a cancelling publisher", - ), - ( - bot_path, - bot_text.replace(bot_line, " group: ${{ github.workflow }}-${{ github.ref }}"), - 1, - "a ref-keyed merge-bot", - ), - ( - bot_path, - bot_text.replace("cancel-in-progress: false", "cancel-in-progress: true"), - 1, - "a cancelling merge-bot", - ), - ( - bot_path, - bot_text.replace(bot_line, bot_line + "${{ github.run_id }}"), - 1, - "a run_id suffix is the publisher's alone", - ), - ( - pub_path, - re.sub(r"(?s)\nconcurrency:.*?cancel-in-progress: \w+\n", "\n", pub_text), - 1, - "no concurrency block", - ), - ( - pub_path, - pub_text.replace( - "github.actor != 'dependabot[bot]'", "github.actor == 'dependabot[bot]'" - ), - 1, - "an actor predicate that is not an exclusion list", - ), - ( - pub_path, - pub_text.replace("refs/heads/main", "refs/heads/develop"), - 1, - "a suffix keyed on another branch", - ), - ( - pub_path, - pub_text.replace("github.actor != 'dependabot[bot]'", "github.actor != 'other[bot]'"), - 1, - "an actor list that is not the plan task's allowlist", - ), - ( - pub_path, - pub_text.replace("ptr727-codegen[bot]", "ptr727-codegen [bot]"), - 1, - "whitespace inside an allowlisted actor name", - ), - ( - bot_path, - bot_text.replace(bot_line, bot_line + pub_value.split("github.ref }}", 1)[1]), - 1, - "the push-only suffix is the publisher's alone", - ), - ( - bot_path, - bot_text.replace(bot_line, bot_line + sched_line.split("github.ref }}", 1)[1]), - 1, - "the schedule-only suffix is the publisher's alone", - ), - ( - pub_path, - pub_text.replace(pub_line, " group: '" + pub_value.replace("'", "''") + "'"), - 0, - "a single-quoted publisher group", - ), - ( - pub_path, - pub_text.replace(pub_line, " group: " + pub_value + " # note"), - 0, - "a trailing comment", - ), - ( - pub_path, - pub_text.replace("concurrency:\n", "concurrency: # note\n", 1), - 0, - "a comment on the key line", - ), - ( - pub_path, - pub_text.replace(pub_line, " group: ${{github.workflow}}-${{github.ref}}"), - 0, - "tight expression spacing", - ), - ( - pub_path, - pub_text.replace("cancel-in-progress: false", "cancel-in-progress: ${{ false }}"), - 0, - "an expression false", - ), - ( - pub_path, - pub_text.replace(pub_line, " group: >-\n " + pub_value), - 1, - "a folded scalar is refused", - ), - ( - pub_path, - pub_text.replace(pub_line, " " + pub_group + "${{ github.run_id }}"), - 1, - "an unconditional run_id suffix", - ), - ( - pub_path, - pub_text.replace( - pub_line, - " " + pub_group + "${{ github.ref == 'refs/heads/main' && github.run_id }}", - ), - 1, - "a suffix naming no event_name", - ), - ( - pub_path, - pub_text.replace( - pub_line, - " " + pub_group + "${{ github.sha }}-${{ github.event_name }}${{ github.run_id }}", - ), - 1, - "several suffix expressions", - ), - ( - bot_path, - "concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }}\n", - 0, - "a bare scalar merge-bot group", - ), - ( - pub_path, - "concurrency: ${{ github.workflow }}-${{ github.ref }}\njobs:\n x:\n concurrency:\n group: y\n", - 0, - "a bare scalar group, job-level concurrency ignored", - ), - ( - ".github/workflows/test-pull-request.yml", - "concurrency:\n group: x\n", - 0, - "another workflow is not read", - ), - ] - refusal_cases = [ - ( - pub_path, - pub_text.replace(pub_line, sched_line), - "cannot read its schedule", - "a schedule-only suffix names what keeps it from qualifying", - ), - ( - bot_path, - bot_text.replace(bot_line, bot_line + sched_line.split("github.ref }}", 1)[1]), - "group is", - "a merge-bot schedule suffix gets no publisher advice", - ), - ( - pub_path, - "concurrency: ${{ github.workflow }}-${{ github.ref }}\n -extra\njobs:\n x: {}\n", - "cannot read", - "a continuation after an inline scalar", - ), - ( - pub_path, - pub_text.replace(pub_line, " group: >-\n " + pub_value), - "cannot read", - "a folded scalar", - ), - ( - pub_path, - pub_text.replace(pub_line, pub_line + "\n more"), - "cannot read", - "a continuation line", - ), - (pub_path, "concurrency: { group: x }\n", "cannot read", "a flow mapping"), - ( - pub_path, - pub_text.replace("concurrency:\n", "concurrency: &c\n", 1), - "", - "an anchor on the key", - ), - ( - pub_path, - pub_text.replace( - pub_line, - " " - + pub_group - + "${{ github.event_name == 'push' && format('-{0}', github.run_id) || '' }}", - ), - "group is", - "a suffix without the main clause", - ), - ( - pub_path, - pub_text.replace( - pub_line, - " " + pub_group + "${{ format('-{0}{1}', github.run_id, github.event_name) }}", - ), - "group is", - "an unconditional suffix naming event_name", - ), - ] - for path_in, text_in, needle, label in refusal_cases: - got = concurrency_caller_findings(path_in, text_in) - if (needle == "" and got) or (needle and not any(needle in f[1] for f in got)): - ok = False - concurrency_ok = False - print(f" FAIL concurrency_caller_findings [{label}] -> {got}") - for path_in, text_in, expected, label in concurrency_cases: - got = len(concurrency_caller_findings(path_in, text_in)) - if got != expected: - ok = False - concurrency_ok = False - print( - f" FAIL concurrency_caller_findings [{label}] -> {got} finding(s), expected {expected}" - ) - if concurrency_ok: - print( - " ok concurrency_caller_findings: the publisher and merge-bot groups match the WORKFLOW.md pins" - ) - - id_cases = [ - ("https://github.com/owner/Repo", "owner/repo"), - ("https://github.com/owner/Repo/", "owner/repo"), - # Padding still resolves here, one-directionally: the gate refuses it, and a repo whose entry is padded must - # read as an entry that exists rather than as a repo with none, per the membership case below. - (" https://github.com/owner/Repo ", "owner/repo"), - ("https://github.com/owner/Repo.git", "owner/repo"), - ("https://github.com/owner/Repo.github", "owner/repo.github"), - ("https://github.com/owner/Repo?tab=readme", None), - ("https://github.com/owner/Repo#readme", None), - ("https://github.com/owner/Repo.git?tab=readme", None), - ("https://github.com/owner?tab=readme/Repo", None), - ("https://github.com/owner#readme/Repo", None), - ("http://github.com/owner/Repo", None), - ("https://gitlab.com/owner/Repo", None), - ("not a url", None), - (None, None), - (42, None), - ] - id_ok = all(repo_identity(url) == want for url, want in id_cases) - if id_ok: - print( - " ok repo_identity: parses owner/repo from a github.com URL, lowercased and trimmed" - ) - else: - ok = False - got_ids = [(url, repo_identity(url)) for url, _ in id_cases] - print(f" FAIL repo_identity -> {got_ids}") - - # owner_repos: forks are dropped, and pagination stops the moment a page comes back short of 100. - real_gh = globals()["gh"] - try: - pages = { - 1: [{"name": f"r{i}", "fork": False} for i in range(100)], - 2: [{"name": "kept", "fork": False}, {"name": "dropped-fork", "fork": True}], - } - globals()["gh"] = lambda path, ok404=False: ( - {"login": "owner"} if path == "user" else pages[int(path.rsplit("page=", 1)[1])] - ) - got = {r["name"] for r in owner_repos("owner")} - finally: - globals()["gh"] = real_gh - want = {f"r{i}" for i in range(100)} | {"kept"} - if got != want: - ok = False - print( - f" FAIL owner_repos -> {len(got)} repos, expected {len(want)}, forks/pagination wrong" - ) - else: - print(" ok owner_repos: forks dropped, pagination stops on a short page") - - # A gh("user") call that returns None (an empty response body, per its own docstring) must - # raise the clear ownership-mismatch error, not crash with AttributeError on .get(). - try: - globals()["gh"] = lambda path, ok404=False: None if path == "user" else [] - owner_repos("owner") - ok = False - print(" FAIL owner_repos: a None gh('user') response did not raise") - except AttributeError: - ok = False - print(" FAIL owner_repos: a None gh('user') response crashed with AttributeError") - except RuntimeError as exc: - # Any RuntimeError would pass a bare except clause, including an unrelated one this test - # was never meant to exercise. Assert the actual ownership-mismatch text. - if "not registry owner 'owner'" not in str(exc): - ok = False - print(f" FAIL owner_repos: a None gh('user') response raised the wrong error -> {exc}") - else: - print(" ok owner_repos: a None gh('user') response raises the clear mismatch error") - finally: - globals()["gh"] = real_gh - - # GitHub logins are case-insensitive, so a registry owner spelled with different casing than - # gh's own login must still be accepted as the same account. - try: - globals()["gh"] = lambda path, ok404=False: {"login": "Owner"} if path == "user" else [] - owner_repos("owner") - print(" ok owner_repos: a differently-cased login still matches the registry owner") - except RuntimeError as e: - ok = False - print(f" FAIL owner_repos: a differently-cased login was wrongly rejected -> {e}") - finally: - globals()["gh"] = real_gh - - # A None page mid-pagination (an empty response body) must fail loud rather than being read - # as a short page: `or []` there would silently truncate the sweep and report clean. - try: - globals()["gh"] = lambda path, ok404=False: {"login": "owner"} if path == "user" else None - owner_repos("owner") - ok = False - print(" FAIL owner_repos: a None page did not raise") - except RuntimeError: - print(" ok owner_repos: a None page fails loud instead of reading as an empty page") - finally: - globals()["gh"] = real_gh - - real_owner_repos = globals()["owner_repos"] - try: - cases = [ - ( - "a repo missing from the registry is a DEFECT", - [{"name": "New", "full_name": "owner/New", "archived": False}], - {"owner": "owner", "repos": []}, - [("DEFECT", "owner/New: exists on GitHub with no registry/repos.json entry")], - ), - ( - "archived on GitHub but not in the registry is DRIFT", - [{"name": "Old", "full_name": "owner/Old", "archived": True}], - { - "owner": "owner", - "repos": [ - { - "name": "Old", - "url": "https://github.com/owner/Old", - "status": "cataloged", - } - ], - }, - [ - ( - "DRIFT", - "Old: archived on GitHub but registry status is 'cataloged' (reconcile to 'archived')", - ) - ], - ), - ( - "registry says archived but GitHub disagrees is DRIFT", - [{"name": "Back", "full_name": "owner/Back", "archived": False}], - { - "owner": "owner", - "repos": [ - { - "name": "Back", - "url": "https://github.com/owner/Back", - "status": "archived", - } - ], - }, - [ - ( - "DRIFT", - "Back: registry status is 'archived' but GitHub reports it unarchived " - + "(reconcile status, or verify it was not unarchived by mistake)", - ) - ], - ), - ( - "matched, not archived either side, is clean", - [{"name": "Fine", "full_name": "owner/Fine", "archived": False}], - { - "owner": "owner", - "repos": [ - { - "name": "Fine", - "url": "https://github.com/owner/Fine", - "status": "cataloged", - } - ], - }, - [], - ), - ( - "identity matching is case-insensitive", - [{"name": "MixedCase", "full_name": "Owner/MixedCase", "archived": False}], - { - "owner": "owner", - "repos": [ - { - "name": "mixedcase", - "url": "https://github.com/owner/mixedcase", - "status": "cataloged", - } - ], - }, - [], - ), - ( - "an incidental URL whitespace difference does not miss a real match", - [{"name": "Spacey", "full_name": "owner/Spacey", "archived": False}], - { - "owner": "owner", - "repos": [ - { - "name": "Spacey", - "url": " https://github.com/owner/Spacey ", - "status": "cataloged", - } - ], - }, - [], - ), - ( - "a same-named repo under a different owner is not treated as a match", - [{"name": "Shared", "full_name": "owner/Shared", "archived": False}], - { - "owner": "owner", - "repos": [ - { - "name": "Shared", - "url": "https://github.com/someone-else/Shared", - "status": "cataloged", - } - ], - }, - [("DEFECT", "owner/Shared: exists on GitHub with no registry/repos.json entry")], - ), - ] - for label, gh_repos, spec_registry, want in cases: - globals()["owner_repos"] = lambda owner, r=gh_repos: r - got = membership_findings({"registry": spec_registry}) - if got != want: - ok = False - print(f" FAIL membership: {label} -> {got}") - else: - print(f" ok membership: {label}") - finally: - globals()["owner_repos"] = real_owner_repos - - # An intentRef with an anchor resolves to the whole hub file, not the anchor-qualified name git cannot look up, and reference still wins where the manifest sets both. - canonical_cases = [ - ( - "no reference or intentRef falls back to the file's own path", - {}, - "AGENTS.md", - "AGENTS.md", - ), - ( - "an intentRef equal to the path is itself the canonical", - {"intentRef": "GOVERNANCE.md"}, - "GOVERNANCE.md", - "GOVERNANCE.md", - ), - ( - "an anchored intentRef strips the anchor and keeps the whole file", - {"intentRef": "GOVERNANCE.md#line-endings"}, - ".editorconfig", - "GOVERNANCE.md", - ), - ( - "reference wins over intentRef when the manifest sets both", - {"reference": "catalog/snippets/configs/codecov.yml", "intentRef": "WORKFLOW.md"}, - "codecov.yml", - "catalog/snippets/configs/codecov.yml", - ), - ( - "a literal '#' in reference is preserved rather than treated as an anchor", - {"reference": "docs/notes#1.md"}, - "notes.md", - "docs/notes#1.md", - ), - ( - "a literal '#' in path is preserved rather than treated as an anchor", - {}, - "docs/notes#1.md", - "docs/notes#1.md", - ), - ( - "a non-string intentRef falls back to path instead of crashing", - {"intentRef": 123}, - "AUDIT.md", - "AUDIT.md", - ), - ( - "an anchor-only intentRef falls back to path rather than an empty canonical", - {"intentRef": "#line-endings"}, - "AUDIT.md", - "AUDIT.md", - ), - ] - for label, item, path, want in canonical_cases: - got = intent_canonical_rel(item, path) - good = got == want - if not good: - ok = False - print(f" {'ok ' if good else 'FAIL'} intent_canonical_rel: {label} -> {got!r}") - + ok = True + for label, got, want in cases: + ok = ok and got == want + print(f" {'ok ' if got == want else 'FAIL'} {label}: got {got!r}") print("SELFTEST PASS" if ok else "SELFTEST FAIL") return 0 if ok else 1 diff --git a/tests/test_audit.py b/tests/test_audit.py index 2dc957579..b9b8ed939 100755 --- a/tests/test_audit.py +++ b/tests/test_audit.py @@ -4,10 +4,7 @@ from __future__ import annotations import base64 -import contextlib -import io import pathlib -import subprocess import sys import unittest from typing import Any @@ -107,7 +104,7 @@ def run_audit( class AuditRepoAlternativesCase(unittest.TestCase): """audit_repo's baseline loop passes each path's declared alternatives into the shared reader. - The reader's own handling of alternatives is covered by the engine's --selftest, so what this + The reader's own handling of alternatives is covered by tests/test_audit_helpers.py, so what this adds is the call site. Were it to stop passing them, a repository holding only an alternative would read as missing the declared path while that self-test still passed. """ @@ -138,7 +135,7 @@ def test_a_repository_holding_neither_is_reported_under_both_names(self) -> None class AuditRepoNamingCase(unittest.TestCase): """audit_repo hands the tree it read, modes included, to the file-naming and secret-stem checks. - Each check's own cases are covered by the engine's --selftest, so what this adds is the call + Each check's own cases are covered by tests/test_audit_helpers.py, so what this adds is the call site: the modes reaching the file-naming check, and its shebang read going to the audited branch. """ @@ -196,80 +193,5 @@ def test_a_failed_shebang_read_raises_rather_than_reading_as_undecided(self) -> run_audit(tree, {"bin/Run_Backup": "100755"}, frozenset({"bin/Run_Backup"})) -def _probe_run(add: Any, commit: Any = None, control: Any = None) -> Any: - """A subprocess.run stand-in answering the invalid-byte probe's own git add of the bad name with `add`.""" - real_run = subprocess.run - - def run(cmd: list[str], *args: Any, **kwargs: Any) -> Any: - cwd = kwargs.get("cwd") - if control is not None and cmd == ["git", "add", "--", "control.txt"]: - return control(cmd) - if cmd[:3] == ["git", "add", "--"] and cwd and cmd[3].startswith("bad-"): - return add(cmd) - if commit is not None and cmd[:2] == ["git", "commit"] and "add invalid-utf8 name" in cmd: - return commit(cmd) - return real_run(cmd, *args, **kwargs) - - return run - - -def _fail(cmd: list[str]) -> Any: - raise subprocess.CalledProcessError(128, cmd) - - -class TestSelftestInvalidByteProbe(unittest.TestCase): - def _selftest(self, run: Any, windows: bool) -> tuple[int, str]: - out = io.StringIO() - with ( - mock.patch.object(audit, "_IS_WINDOWS", windows), - mock.patch.object(subprocess, "run", run), - contextlib.redirect_stdout(out), - ): - result = audit._selftest() - return result, out.getvalue() - - def _require_probe_file(self, text: str) -> None: - if "host cannot create it" in text or "no candidate byte" in text: - self.skipTest("this host cannot exercise the invalid-byte probe") - - def test_a_failing_add_of_the_probe_file_skips_on_windows(self) -> None: - result, text = self._selftest(_probe_run(_fail), windows=True) - self._require_probe_file(text) - self.assertIn("skip hub_tracked: non-UTF-8 filename (git cannot add it)", text) - self.assertEqual(result, 0, text) - - def _error_of(self, run: Any, windows: bool) -> subprocess.CalledProcessError: - out = io.StringIO() - with ( - mock.patch.object(audit, "_IS_WINDOWS", windows), - mock.patch.object(subprocess, "run", run), - contextlib.redirect_stdout(out), - ): - try: - audit._selftest() - except subprocess.CalledProcessError as error: - return error - self._require_probe_file(out.getvalue()) - self.fail("the git failure did not propagate") - - def test_a_failing_control_add_fails_loudly_on_windows(self) -> None: - error = self._error_of( - _probe_run(lambda cmd: subprocess.CompletedProcess(cmd, 0), control=_fail), - windows=True, - ) - self.assertEqual(error.cmd, ["git", "add", "--", "control.txt"]) - - def test_a_failing_add_of_the_probe_file_fails_loudly_elsewhere(self) -> None: - error = self._error_of(_probe_run(_fail), windows=False) - self.assertEqual(error.cmd[:3], ["git", "add", "--"]) - - def test_another_git_failure_still_fails_loudly(self) -> None: - run = _probe_run( - lambda cmd: subprocess.CompletedProcess(cmd, 0), commit=lambda cmd: _fail(cmd) - ) - error = self._error_of(run, windows=True) - self.assertEqual(error.cmd[:2], ["git", "commit"]) - - if __name__ == "__main__": unittest.main() diff --git a/tests/test_audit_helpers.py b/tests/test_audit_helpers.py new file mode 100755 index 000000000..659047d1f --- /dev/null +++ b/tests/test_audit_helpers.py @@ -0,0 +1,4311 @@ +#!/usr/bin/env python3 +"""spec/audit.py's helpers on constructed fixtures, with no network read.""" + +from __future__ import annotations + +import contextlib +import io +import locale +import pathlib +import re +import subprocess +import sys +import tempfile +import time +import unittest +from collections.abc import Callable +from typing import Any +from unittest import mock + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent / "spec")) +import audit +from git_env import git_location_cleared + +_LOCATION = contextlib.ExitStack() + + +def setUpModule() -> None: + # A git hook sets GIT_DIR, which would otherwise place every temporary repository inside the hook's own. + _LOCATION.enter_context(git_location_cleared()) + + +def tearDownModule() -> None: + _LOCATION.close() + + +PR_CHECK = " check-workflow-status:\n name: Check pull request workflow status job\n needs: [changes]\n runs-on: ubuntu-latest\n" +PR_HEAD = "name: Test\non: pull_request\njobs:\n changes:\n runs-on: ubuntu-latest\n" +PR_CONTRACT = { + "requiredJobKeys": ["check-workflow-status"], + "requiredCheckName": "Check pull request workflow status job", +} +# The validate-task.yml stub contract: a caller's validate job must exist and reach the hub task by name. +# A token check alone would pass a stub that drops the validate job entirely, since it only runs for a job that is present. +# Naming the validate job in requiredJobKeys too catches a dropped job on its own. +PR_STUB_CONTRACT = dict( + PR_CONTRACT, + requiredJobKeys=["check-workflow-status", "validate"], + requireTokensInJob={"validate": ["validate-task.yml"]}, +) +PR_VALIDATE_HEAD = ( + "name: Test\non: pull_request\njobs:\n" + " validate:\n" + " name: Validate sources job\n" + " uses: ptr727/ProjectTemplate/.github/workflows/validate-task.yml@" + + "a" * 40 + + " # 2.0.1\n" +) +PR_VALIDATE_INLINE = ( + "name: Test\non: pull_request\njobs:\n" + " validate:\n" + " name: Validate sources job\n" + " runs-on: ubuntu-latest\n" + " steps:\n - run: echo inline lint\n" +) +# A validate-task.yml stub's own aggregator needs the validate job, never the release-shape changes job. +# The shared PR_CHECK fixture needs a changes job these two fixtures do not define, so this one needs validate instead. +PR_CHECK_VALIDATE = " check-workflow-status:\n name: Check pull request workflow status job\n needs: [validate]\n runs-on: ubuntu-latest\n" +GH_REL = ( + " github-release:\n needs: [get-version, build-widget]\n runs-on: ubuntu-latest\n steps:\n" + " - uses: actions/download-artifact@v4\n with:\n pattern: release-asset-${{ inputs.branch }}-*\n merge-multiple: true\n" +) +REL_HEAD = ( + "name: Build Release\non:\n workflow_call:\njobs:\n" + " get-version:\n runs-on: ubuntu-latest\n steps: []\n" + " build-widget:\n runs-on: ubuntu-latest\n steps: []\n" +) +REL_OK = REL_HEAD + GH_REL +REL_CONTRACT = { + "requiredJobKeys": ["get-version", "github-release"], + "artifactNameToken": "release-asset-", + "requireTokensInJob": {"github-release": ["pattern:", "merge-multiple:"]}, + "forbidTokensInJob": {"github-release": ["artifact-ids:"]}, +} +# The merge-bot caller stub, the shape every repo carries once the merge-bot is hub-hosted. +BOT_STUB = ( + "jobs:\n" + " merge-bot:\n" + " name: Merge bot pull request job\n" + " uses: acme/hub/.github/workflows/merge-bot-task.yml@" + "a" * 40 + " # 2.0.1\n" + " secrets:\n" + " CODEGEN_APP_CLIENT_ID: ${{ secrets.CODEGEN_APP_CLIENT_ID }}\n" + " CODEGEN_APP_PRIVATE_KEY: ${{ secrets.CODEGEN_APP_PRIVATE_KEY }}\n" +) +BOT_CONTRACT = { + "requiredJobKeys": ["merge-bot"], + "requireTokensInJob": { + "merge-bot": ["merge-bot-task.yml", "CODEGEN_APP_CLIENT_ID", "CODEGEN_APP_PRIVATE_KEY"] + }, +} +# The publish-release.yml caller stub once the release chain is hub-hosted, plan, validate and publish job keys. +# Each names its hub task by token, per docs/reusable-workflows.md "Adopting the Release Chain". +# Validate's own token requirement is what stops a stub from satisfying the job key with a no-op job. +PUBLISH_STUB = ( + "jobs:\n" + " plan:\n" + " name: Plan release job\n" + " uses: acme/hub/.github/workflows/publish-plan-task.yml@" + "a" * 40 + " # 2.0.1\n" + " with:\n" + " event_name: ${{ github.event_name }}\n" + " validate:\n" + " name: Validate sources job\n" + " needs: [plan]\n" + " if: ${{ needs.plan.outputs.publish == 'true' }}\n" + " uses: ./.github/workflows/validate-task.yml\n" + " publish:\n" + " name: Publish project release job\n" + " needs: [plan, validate]\n" + " if: ${{ needs.plan.outputs.publish == 'true' && needs.validate.result == 'success' }}\n" + " uses: acme/hub/.github/workflows/build-release-task.yml@" + "a" * 40 + " # 2.0.1\n" + " with:\n" + " github: true\n" +) +PUBLISH_CONTRACT = { + "requiredJobKeys": ["plan", "validate", "publish"], + "requireTokensInJob": { + "plan": ["publish-plan-task.yml"], + "validate": ["validate-task.yml"], + "publish": ["build-release-task.yml", "needs.validate.result == 'success'"], + }, +} +INTERFACE_CASES = [ + ("conformant PR workflow", PR_HEAD + PR_CHECK, PR_CONTRACT, 0), + ("PR workflow missing the required job and its check name", PR_HEAD, PR_CONTRACT, 2), + ( + "PR workflow with a renamed check", + PR_HEAD + PR_CHECK.replace("Check pull request workflow status job", "Renamed"), + PR_CONTRACT, + 1, + ), + ( + "check name present only in a run step, not as a job name", + PR_HEAD + + " check-workflow-status:\n runs-on: ubuntu-latest\n steps:\n - run: echo Check pull request workflow status job\n", + PR_CONTRACT, + 1, + ), + ( + "check name present only as a step name, not the job name", + PR_HEAD + + " check-workflow-status:\n runs-on: ubuntu-latest\n steps:\n - name: Check pull request workflow status job\n run: true\n", + PR_CONTRACT, + 1, + ), + ( + "PR stub validate job reaching the hub validate-task", + PR_VALIDATE_HEAD + PR_CHECK_VALIDATE, + PR_STUB_CONTRACT, + 0, + ), + ( + "PR stub validate job still carrying an inline lint job", + PR_VALIDATE_INLINE + PR_CHECK_VALIDATE, + PR_STUB_CONTRACT, + 1, + ), + ( + "PR stub dropping the validate job entirely", + PR_HEAD + PR_CHECK, + PR_STUB_CONTRACT, + 1, + ), + ("conformant release task", REL_OK, REL_CONTRACT, 0), + ( + "release task with an artifact-ids fork in github-release", + REL_OK.replace( + " merge-multiple: true\n", + " merge-multiple: true\n artifact-ids: 123\n", + ), + REL_CONTRACT, + 1, + ), + ( + "release task missing merge-multiple in github-release", + REL_OK.replace(" merge-multiple: true\n", ""), + REL_CONTRACT, + 1, + ), + ( + "release task with an owned extra leaf job", + REL_HEAD + " build-extra:\n runs-on: ubuntu-latest\n steps: []\n" + GH_REL, + REL_CONTRACT, + 0, + ), + ( + "absent job reports once, no redundant token findings", + REL_HEAD, + { + "requiredJobKeys": ["github-release"], + "requireTokensInJob": {"github-release": ["pattern:", "merge-multiple:"]}, + }, + 1, + ), + ( + "a forbidden token only in a comment is ignored", + REL_OK.replace( + " merge-multiple: true\n", + " merge-multiple: true\n # never an artifact-ids: fork here\n", + ), + REL_CONTRACT, + 0, + ), + ( + "a required token only in a comment does not count", + REL_OK.replace( + " merge-multiple: true\n", " # merge-multiple: true (was here)\n" + ), + REL_CONTRACT, + 1, + ), + ( + "merge-bot caller stub reaching the hub task with both secrets mapped", + BOT_STUB, + BOT_CONTRACT, + 0, + ), + ( + "merge-bot caller stub with the hub's own local uses", + BOT_STUB.replace( + "acme/hub/.github/workflows/merge-bot-task.yml@" + "a" * 40 + " # 2.0.1", + "./.github/workflows/merge-bot-task.yml", + ), + BOT_CONTRACT, + 0, + ), + ( + "merge-bot copy still carrying the job bodies reports the missing caller job once", + ( + "jobs:\n merge-dependabot:\n runs-on: ubuntu-latest\n steps: []\n" + " disable-auto-merge-on-maintainer-push:\n runs-on: ubuntu-latest\n steps: []\n" + ), + BOT_CONTRACT, + 1, + ), + ( + "merge-bot caller stub that maps only one secret", + BOT_STUB.replace( + " CODEGEN_APP_PRIVATE_KEY: ${{ secrets.CODEGEN_APP_PRIVATE_KEY }}\n", "" + ), + BOT_CONTRACT, + 1, + ), + ( + "publish-release.yml stub reaching both hub tasks by pin", + PUBLISH_STUB, + PUBLISH_CONTRACT, + 0, + ), + ( + "publish-release.yml stub with the hub's own local uses", + PUBLISH_STUB.replace( + "acme/hub/.github/workflows/publish-plan-task.yml@" + "a" * 40 + " # 2.0.1", + "./.github/workflows/publish-plan-task.yml", + ).replace( + "acme/hub/.github/workflows/build-release-task.yml@" + "a" * 40 + " # 2.0.1", + "./.github/workflows/build-release-task.yml", + ), + PUBLISH_CONTRACT, + 0, + ), + ( + "publish-release.yml stub missing the plan job reports it once", + "jobs:\n" + " validate:\n" + " name: Validate sources job\n" + " uses: ./.github/workflows/validate-task.yml\n" + " publish:\n" + " name: Publish project release job\n" + " needs: [validate]\n" + " if: ${{ needs.validate.result == 'success' }}\n" + " uses: acme/hub/.github/workflows/build-release-task.yml@" + "a" * 40 + " # 2.0.1\n", + PUBLISH_CONTRACT, + 1, + ), + ( + "publish-release.yml stub missing the validate job reports it once", + "jobs:\n" + " plan:\n" + " name: Plan release job\n" + " uses: acme/hub/.github/workflows/publish-plan-task.yml@" + "a" * 40 + " # 2.0.1\n" + " publish:\n" + " name: Publish project release job\n" + " needs: [plan, validate]\n" + " if: ${{ needs.validate.result == 'success' }}\n" + " uses: acme/hub/.github/workflows/build-release-task.yml@" + "a" * 40 + " # 2.0.1\n", + PUBLISH_CONTRACT, + 1, + ), + ( + "publish-release.yml stub whose publish job never names build-release-task.yml", + PUBLISH_STUB.replace( + "acme/hub/.github/workflows/build-release-task.yml@" + "a" * 40 + " # 2.0.1", + "./.github/workflows/build-release-task-renamed.yml", + ), + PUBLISH_CONTRACT, + 1, + ), + ( + "publish-release.yml stub whose validate job never names validate-task.yml", + PUBLISH_STUB.replace( + " uses: ./.github/workflows/validate-task.yml\n", + " uses: ./.github/workflows/validate-task-renamed.yml\n", + ), + PUBLISH_CONTRACT, + 1, + ), + ( + "publish-release.yml stub carrying its publish condition in a folded if: >-", + PUBLISH_STUB.replace( + " if: ${{ needs.plan.outputs.publish == 'true' && needs.validate.result == 'success' }}\n", + " if: >-\n" + " ${{ needs.plan.outputs.publish == 'true'\n" + " && needs.validate.result == 'success' }}\n", + ), + PUBLISH_CONTRACT, + 0, + ), + ( + "publish-release.yml stub whose folded if: >- wraps inside the required token", + PUBLISH_STUB.replace( + " if: ${{ needs.plan.outputs.publish == 'true' && needs.validate.result == 'success' }}\n", + " if: >-\n" + " ${{ needs.plan.outputs.publish == 'true' && needs.validate.result\n" + " == 'success' }}\n", + ), + PUBLISH_CONTRACT, + 0, + ), + ( + "publish-release.yml stub naming the validation token only in a name: | body", + PUBLISH_STUB.replace( + " name: Publish project release job\n", + " name: |\n needs.validate.result == 'success'\n", + ).replace(" && needs.validate.result == 'success'", ""), + PUBLISH_CONTRACT, + 1, + ), + ( + "publish-release.yml stub whose publish job ignores failed validation", + PUBLISH_STUB.replace(" && needs.validate.result == 'success'", ""), + PUBLISH_CONTRACT, + 1, + ), +] +# The deploy-site.yml caller stub once deploy-site-task.yml is hub-hosted: no secrets: inherit, the required crossing secret named instead. +# GitHub documents secrets: inherit for same-org or enterprise callers, so the fleet does not use it across repositories. +# No job-level environment: on the caller, unsupported on a job with uses:. +DEPLOY_STUB = ( + "jobs:\n" + " assert-ref:\n runs-on: ubuntu-latest\n steps: []\n" + " validate:\n uses: ./.github/workflows/validate-task.yml\n" + " deploy:\n" + " name: Deploy job\n" + " permissions:\n contents: read\n" + " uses: acme/hub/.github/workflows/deploy-site-task.yml@" + "a" * 40 + " # 2.0.1\n" + " with:\n environment: ${{ inputs.environment }}\n" + " secrets:\n DEPLOY_SSH_PRIVATE_KEY: ${{ secrets.DEPLOY_SSH_PRIVATE_KEY }}\n" +) +DEPLOY_CONTRACT = { + "requiredJobKeys": ["assert-ref", "validate", "deploy"], + "requireTokensInJob": { + "deploy": [ + "deploy-site-task.yml", + # Anchored to the with: block specifically, not just 6-space indent. + # A secret named "environment" would also land at that indent otherwise. + # The task declares this input required, and a missing one fails at dispatch, not at audit time. + "with:\n environment:", + "contents: read", + "DEPLOY_SSH_PRIVATE_KEY", + ] + }, + "forbidTokensInJob": { + # Indent-anchored to the job's own top level, the shape GitHub rejects on a job with uses:. + "deploy": ["\n environment:"] + }, +} +INTERFACE_CASES += [ + ( + "deploy-site.yml caller stub reaching the hub task with the crossing secret mapped", + DEPLOY_STUB, + DEPLOY_CONTRACT, + 0, + ), + ( + "deploy-site.yml caller stub still using secrets: inherit reports the missing secret", + DEPLOY_STUB.replace( + " secrets:\n DEPLOY_SSH_PRIVATE_KEY: ${{ secrets.DEPLOY_SSH_PRIVATE_KEY }}\n", + " secrets: inherit\n", + ), + DEPLOY_CONTRACT, + 1, + ), + ( + "deploy-site.yml caller stub reintroducing the invalid job-level environment: key", + DEPLOY_STUB.replace( + " name: Deploy job\n", + " name: Deploy job\n environment: ${{ inputs.environment }}\n", + ), + DEPLOY_CONTRACT, + 1, + ), + ( + "deploy-site.yml caller stub missing the required environment input", + DEPLOY_STUB.replace(" with:\n environment: ${{ inputs.environment }}\n", ""), + DEPLOY_CONTRACT, + 1, + ), + ( + "deploy-site.yml caller stub with environment: under the wrong mapping still reports missing", + # A same-indented environment: key under secrets: must not satisfy the with:-anchored requirement on its own. + # The oddly-named secret's value is left generic so it does not incidentally satisfy the DEPLOY_SSH_PRIVATE_KEY token. + DEPLOY_STUB.replace( + " with:\n environment: ${{ inputs.environment }}\n", + " secrets:\n environment: ${{ secrets.SOME_OTHER_SECRET }}\n", + ), + DEPLOY_CONTRACT, + 1, + ), + ( + "deploy-site.yml caller stub missing the contents: read grant the task's jobs declare", + DEPLOY_STUB.replace(" permissions:\n contents: read\n", ""), + DEPLOY_CONTRACT, + 1, + ), + ( + # A block scalar crafted to contain the required token as string content must still report it missing. + # The real `with:` mapping is removed here, so only the block scalar carries the text. + "deploy-site.yml caller stub with the with:/environment: tokens only inside a block-scalar name still reports missing", + DEPLOY_STUB.replace( + " name: Deploy job\n", " name: |\n with:\n environment:\n" + ).replace(" with:\n environment: ${{ inputs.environment }}\n", ""), + DEPLOY_CONTRACT, + 1, + ), +] +# The stage-5 type-specific hub tasks carry no manifest entry yet, since no repo has adopted a caller stub for them. +# These fixtures exercise the contract adoption will register, proving the interface engine reads it correctly before any downstream repo depends on that reading. +README_STUB = ( + "jobs:\n" + " publish-docker-readme:\n" + " name: Publish Docker Hub readme job\n" + " permissions:\n contents: read\n" + " uses: acme/hub/.github/workflows/publish-docker-readme-task.yml@" + "a" * 40 + " # 2.0.1\n" + " with:\n branch: ${{ github.ref_name }}\n" + " secrets:\n" + " DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}\n" + " DOCKER_HUB_ACCESS_TOKEN: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}\n" +) +README_CONTRACT = { + "requiredJobKeys": ["publish-docker-readme"], + "requireTokensInJob": { + "publish-docker-readme": [ + "publish-docker-readme-task.yml", + "contents: read", + "DOCKER_HUB_USERNAME", + "DOCKER_HUB_ACCESS_TOKEN", + ] + }, +} +UPSTREAM_STUB = ( + "jobs:\n" + " check-upstream-version:\n" + " name: Check upstream version job\n" + " uses: acme/hub/.github/workflows/check-upstream-version-task.yml@" + + "a" + * 40 + + " # 2.0.1\n" + " secrets:\n" + " CODEGEN_APP_CLIENT_ID: ${{ secrets.CODEGEN_APP_CLIENT_ID }}\n" + " CODEGEN_APP_PRIVATE_KEY: ${{ secrets.CODEGEN_APP_PRIVATE_KEY }}\n" +) +UPSTREAM_CONTRACT = { + "requiredJobKeys": ["check-upstream-version"], + "requireTokensInJob": { + "check-upstream-version": [ + "check-upstream-version-task.yml", + "CODEGEN_APP_CLIENT_ID", + "CODEGEN_APP_PRIVATE_KEY", + ] + }, +} +CODEGEN_STUB = ( + "jobs:\n" + " run-codegen:\n" + " name: Run codegen and pull request job\n" + " uses: acme/hub/.github/workflows/run-codegen-pull-request-task.yml@" + + "a" + * 40 + + " # 2.0.1\n" + " secrets:\n" + " CODEGEN_APP_CLIENT_ID: ${{ secrets.CODEGEN_APP_CLIENT_ID }}\n" + " CODEGEN_APP_PRIVATE_KEY: ${{ secrets.CODEGEN_APP_PRIVATE_KEY }}\n" +) +CODEGEN_CONTRACT = { + "requiredJobKeys": ["run-codegen"], + "requireTokensInJob": { + "run-codegen": [ + "run-codegen-pull-request-task.yml", + "CODEGEN_APP_CLIENT_ID", + "CODEGEN_APP_PRIVATE_KEY", + ] + }, +} +INTERFACE_CASES += [ + case + for prefix, stub, contract in ( + ("publish-docker-readme.yml", README_STUB, README_CONTRACT), + ("check-upstream-version.yml", UPSTREAM_STUB, UPSTREAM_CONTRACT), + ("run-codegen-pull-request.yml", CODEGEN_STUB, CODEGEN_CONTRACT), + ) + for case in ( + (f"{prefix} caller stub reaching the hub task with secrets mapped", stub, contract, 0), + ( + f"{prefix} copy still carrying the job body reports the missing caller job", + "jobs:\n some-other-job:\n runs-on: ubuntu-latest\n steps: []\n", + contract, + 1, + ), + ) +] + +CODE_VIEW_CASES = [ + ( + "block scalar body dropped, key line kept", + " deploy:\n name: |\n with:\n environment:\n", + " deploy:\n name: |", + ), + ( + "a dedented sibling key ends the block scalar body", + " deploy:\n name: |\n with:\n with:\n environment: prod\n", + " deploy:\n name: |\n with:\n environment: prod", + ), + ( + "folded scalar, strip-chomp indicator, trailing comment on the key line", + " deploy:\n name: >- # a folded, strip-chomped scalar\n environment:\n", + " deploy:\n name: >- # a folded, strip-chomped scalar", + ), + ( + # A step's `- run: |` boundary is `run:`'s column, not the dash's column. + # A sibling `env:` key genuinely at that column is real structure, not block-scalar body. + "a step's `- run: |` block scalar body drops, its sibling env: mapping survives", + " - run: |\n echo body\n env:\n TOKEN: xyz\n", + " - run: |\n env:\n TOKEN: xyz", + ), + ( + # A valid YAML anchor property (`&label`) sits between the colon and the indicator. + "an anchored block scalar (`name: &label |`) is still recognized as a block-scalar key", + " deploy:\n name: &lbl |\n with:\n environment:\n", + " deploy:\n name: &lbl |", + ), + ( + # A bare sequence item (a matrix string) has no key, so its boundary is the dash's column. + "a bare sequence-item block scalar (`- |`, a matrix string) still drops its body", + " include:\n - |\n TOKEN\n", + " include:\n - |", + ), + ( + "a folded `if: >-` keeps its body, folded onto the key line", + " publish:\n if: >-\n ${{ a\n\n && b }}\n uses: x\n", + " publish:\n if: >- ${{ a && b }}\n uses: x", + ), + ( + "a step's folded `- if: >-` keeps its body too", + " - if: >-\n ${{ a }}\n run: echo\n", + " - if: >- ${{ a }}\n run: echo", + ), +] + + +class CheckInterfaceCase(unittest.TestCase): + """Holds the check_interface finding counts over synthetic workflow stubs.""" + + def test_finding_counts(self) -> None: + for label, text, contract, want in INTERFACE_CASES: + with self.subTest(label): + self.assertEqual(len(audit.check_interface("wf", contract, text)), want) + + +class SplitJobsCase(unittest.TestCase): + """Holds the split_jobs edge cases.""" + + def test_trailing_comment(self) -> None: + trailing = audit.split_jobs(REL_OK + "# a trailing top-level comment\n") + self.assertEqual(set(trailing), {"get-version", "build-widget", "github-release"}) + self.assertNotIn( + "trailing top-level comment", + trailing.get("github-release", ""), + "split_jobs (trailing comment)", + ) + + def test_inline_mapping(self) -> None: + inline = audit.split_jobs( + "name: X\non: push\njobs:\n quick: {runs-on: ubuntu-latest}\n full:\n runs-on: ubuntu-latest\n" + ) + self.assertEqual(set(inline), {"quick", "full"}) + self.assertIn("runs-on", inline.get("quick", ""), "split_jobs (inline mapping)") + + +class CodeViewCase(unittest.TestCase): + """Holds the block-scalar handling of _code_view.""" + + def test_block_scalar_handling(self) -> None: + for label, text, want in CODE_VIEW_CASES: + with self.subTest(label): + self.assertEqual(audit._code_view(text), want) + + +CANON = "line one\nline two\nline three\n" +PIN_A = " - uses: actions/checkout@" + "a" * 40 + " # v7.0.0\n" +PIN_B = " - uses: actions/checkout@" + "B" * 40 + " # v7.0.1\n" +PIN_STRUCT = " - uses: actions/setup-node@" + "a" * 40 + " # v7.0.0\n" +NOTE_X = " - uses: actions/checkout@" + "a" * 40 + " # kept for the audited build\n" +NOTE_Y = " - uses: actions/checkout@" + "a" * 40 + " # kept for a different reason\n" +D3 = "2024-03-01T00:00:00+00:00" +D2 = "2024-02-01T00:00:00+00:00" +D1 = "2024-01-01T00:00:00+00:00" +IDENTITY = ( + ["git", "config", "user.email", "test@test.invalid"], + ["git", "config", "user.name", "test"], +) + + +class VerbatimCase(unittest.TestCase): + """Holds classify_verbatim and the content_hash normalization checks.""" + + def test_classify_verbatim(self) -> None: + cases = [ + # (label, down_text, canon_text, history, want) + ("identical -> match", CANON, CANON, [], None), + ("EOL-only diff (CRLF) -> match", CANON.replace("\n", "\r\n"), CANON, [], None), + ("EOL-only diff (bare CR) -> match", CANON.replace("\n", "\r"), CANON, [], None), + ( + "body edit -> modified", + CANON.replace("line two", "line TWO edited"), + CANON, + [], + "modified", + ), + ( + "matches a past revision -> stale", + "old body\n", + "current body\n", + ["old body\n", "older\n"], + "stale", + ), + ( + "matches a past revision modulo EOL -> stale", + "old body\r\n", + "current body\n", + ["old body\n"], + "stale", + ), + ( + "edit in no revision -> modified", + "never existed\n", + "current body\n", + ["old body\n"], + "modified", + ), + ] + for label, down, canon_t, history, want in cases: + with self.subTest(label): + self.assertEqual(audit.classify_verbatim(down, canon_t, history), want) + + def test_action_pin_normalization(self) -> None: + self.assertEqual( + audit.content_hash(PIN_A), + audit.content_hash(PIN_B), + "action-pin: a uses:@ bump (sha + # vN, uppercase hex) should normalize equal", + ) + self.assertNotEqual( + audit.content_hash(PIN_A), + audit.content_hash(PIN_STRUCT), + "action-pin: a changed action name must still hash differently", + ) + self.assertNotEqual( + audit.content_hash(NOTE_X), + audit.content_hash(NOTE_Y), + "action-pin: a hand-written (non-version) pin comment must survive to be compared", + ) + + def test_needs_mask(self) -> None: + needs_full = ( + " github-release:\n" + " needs: [get-version, validate-release, build-nuget, dotnet-publish]\n" + " runs-on: x\n steps: []\n" + ) + needs_pruned = ( + " github-release:\n" + " needs: [get-version, validate-release, dotnet-publish]\n" + " runs-on: x\n steps: []\n" + ) + needs_block = ( + " github-release:\n needs:\n - get-version\n - dotnet-publish\n" + " runs-on: x\n steps: []\n" + ) + needs_scalar = ( + " github-release:\n needs: dotnet-publish\n runs-on: x\n steps: []\n" + ) + needs_forked = ( + " github-release:\n needs: [get-version, validate-release]\n" + " runs-on: x\n steps:\n - run: fork\n" + ) + self.assertEqual( + len( + { + audit.content_hash(needs_full), + audit.content_hash(needs_pruned), + audit.content_hash(needs_block), + audit.content_hash(needs_scalar), + } + ), + 1, + "needs-mask: a pruned needs list (inline, block, or scalar) should normalize equal", + ) + self.assertNotEqual( + audit.content_hash(needs_full), + audit.content_hash(needs_forked), + "needs-mask: a step change must still hash differently", + ) + self.assertIn( + "runs-on: x", + audit.normalize(needs_block), + "needs-mask: masking a block needs list must not consume the next key", + ) + + +class LastEffectiveChangeCase(unittest.TestCase): + """Holds _last_effective_change, which skips a normalized-only bump.""" + + def test_last_effective_change(self) -> None: + cases = [ + ( + "every bump back to creation is pin-only -> creation wins", + [(D3, "sha3", PIN_B), (D2, "sha2", PIN_A), (D1, "sha1", PIN_A)], + (D1, "sha1"), + ), + ( + "newest differs for real -> newest wins", + [(D3, "sha3", PIN_STRUCT), (D2, "sha2", PIN_A), (D1, "sha1", PIN_A)], + (D3, "sha3"), + ), + ( + "newest is a pin-only bump over a real change -> the real change wins", + [(D3, "sha3", PIN_B), (D2, "sha2", PIN_A), (D1, "sha1", PIN_STRUCT)], + (D2, "sha2"), + ), + ( + "one revision (creation) -> that revision wins, nothing to compare against", + [(D1, "sha1", "only revision\n")], + (D1, "sha1"), + ), + ( + "unreadable newest text -> treated as effective, never silently skipped", + [(D2, "sha2", None), (D1, "sha1", "whatever\n")], + (D2, "sha2"), + ), + ("no history -> None", [], None), + ] + for label, revisions, want in cases: + with self.subTest(label): + self.assertEqual(audit._last_effective_change(revisions), want) + + +class GitRepoCase(unittest.TestCase): + """Base for cases that build temporary git repositories.""" + + def run_git(self, cwd: pathlib.Path, *args: str) -> None: + subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True) + + def make_dir(self) -> pathlib.Path: + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + return pathlib.Path(tmp.name) + + def make_repo(self, *init_args: str) -> pathlib.Path: + path = self.make_dir() + self.run_git(path, "init", "-q", *init_args) + for cmd in IDENTITY: + self.run_git(path, *cmd[1:]) + return path + + +class GitRevisionsCase(GitRepoCase): + """Holds _git_revisions against temporary git repositories.""" + + def revision_texts(self, root: pathlib.Path, rel: str, **kwargs: str) -> list[str | None]: + with mock.patch.object(audit, "ROOT", root): + audit._git_revisions.cache_clear() + self.addCleanup(audit._git_revisions.cache_clear) + revisions = audit._git_revisions(rel, **kwargs) + return [text for _, _, text in revisions] + + def test_re_added_file(self) -> None: + # A deletion revision reads as None even once rel_path exists again in a later commit. + root = self.make_repo() + rel = "deletion-probe.txt" + (root / rel).write_text("v1\n") + self.run_git(root, "add", rel) + self.run_git(root, "commit", "-q", "-m", "add") + self.run_git(root, "rm", "-q", rel) + self.run_git(root, "commit", "-q", "-m", "delete") + (root / rel).write_text("v2\n") + self.run_git(root, "add", rel) + self.run_git(root, "commit", "-q", "-m", "readd") + self.assertEqual( + self.revision_texts(root, rel, rev="HEAD"), + ["v2\n", None, "v1\n"], + "_git_revisions: re-added file", + ) + + def test_file_to_directory_transition(self) -> None: + root = self.make_repo() + rel = "thing" + (root / rel).write_text("v1\n") + self.run_git(root, "add", rel) + self.run_git(root, "commit", "-q", "-m", "file") + self.run_git(root, "rm", "-q", rel) + (root / rel).mkdir() + (root / rel / "inner.txt").write_text("v2\n") + self.run_git(root, "add", rel) + self.run_git(root, "commit", "-q", "-m", "now a directory") + self.assertEqual( + self.revision_texts(root, rel, rev="HEAD"), + [None, "v1\n"], + "_git_revisions: file-to-directory transition", + ) + + def test_file_to_symlink_transition(self) -> None: + root = self.make_repo() + rel = "thing" + (root / rel).write_text("v1\n") + self.run_git(root, "add", rel) + self.run_git(root, "commit", "-q", "-m", "file") + self.run_git(root, "rm", "-q", rel) + try: + (root / rel).symlink_to("target") + except OSError: + self.skipTest( + "_git_revisions: file-to-symlink transition (host cannot create symlinks)" + ) + self.run_git(root, "add", rel) + self.run_git(root, "commit", "-q", "-m", "now a symlink") + self.assertEqual( + self.revision_texts(root, rel, rev="HEAD"), + [None, "v1\n"], + "_git_revisions: file-to-symlink transition", + ) + + def test_default_rev_reads_origin_main(self) -> None: + # `origin` is a plain local path here, so the fetch stays offline. + upstream = self.make_repo("-b", "main") + root = self.make_dir() + rel = "hub-probe.txt" + (upstream / rel).write_text("main-content\n") + self.run_git(upstream, "add", rel) + self.run_git(upstream, "commit", "-q", "-m", "add on main") + subprocess.run( + ["git", "clone", "-q", str(upstream), str(root)], check=True, capture_output=True + ) + # A clone carries no committer identity of its own, so it needs the same setup as upstream. + for cmd in IDENTITY: + self.run_git(root, *cmd[1:]) + self.run_git(root, "checkout", "-q", "-b", "develop") + (root / rel).write_text("develop-only-content\n") + self.run_git(root, "add", rel) + self.run_git(root, "commit", "-q", "-m", "develop-only change") + with mock.patch.object(audit, "ROOT", root): + audit._git_revisions.cache_clear() + audit._hub_main_rev.cache_clear() + self.addCleanup(audit._git_revisions.cache_clear) + self.addCleanup(audit._hub_main_rev.cache_clear) + revisions = audit._git_revisions(rel) + self.assertEqual( + [text for _, _, text in revisions], + ["main-content\n"], + "_git_revisions: default rev reads origin's main, not the checked-out branch", + ) + + +class HubTrackedCase(GitRepoCase): + """Holds hub_tracked against a filename the active locale rejects.""" + + def test_non_utf8_filename_round_trips(self) -> None: + # The ls-tree output is NUL-delimited raw bytes, so decoding before the NUL-split would crash. + encoding = locale.getpreferredencoding(False) + invalid_bytes = None + for candidate in (b"\xff", b"\x80\x81", b"\xfe\xff"): + try: + candidate.decode(encoding) + except UnicodeDecodeError: + invalid_bytes = candidate + break + if invalid_bytes is None: + self.skipTest(f"hub_tracked: no candidate byte sequence is invalid under {encoding!r}") + bad_name = "bad-" + invalid_bytes.decode("utf-8", errors="surrogateescape") + "-name.txt" + root = self.make_repo("-b", "main") + (root / "control.txt").write_text("x") + self.run_git(root, "add", "--", "control.txt") + try: + (root / bad_name).write_text("x") + except OSError: + self.skipTest("hub_tracked: non-UTF-8 filename (host cannot create it)") + try: + self.run_git(root, "add", "--", bad_name) + except subprocess.CalledProcessError: + if not audit._IS_WINDOWS: + raise + self.skipTest("hub_tracked: non-UTF-8 filename (git cannot add it)") + self.run_git(root, "commit", "-q", "-m", "add invalid-utf8 name") + with mock.patch.object(audit, "ROOT", root): + audit.hub_tracked.cache_clear() + self.addCleanup(audit.hub_tracked.cache_clear) + tracked = audit.hub_tracked(rev="HEAD") + self.assertTrue( + bad_name in tracked, + "hub_tracked: a non-UTF-8 filename round-trips instead of crashing", + ) + + +def _probe_run(add: Any, commit: Any = None, control: Any = None) -> Any: + """A subprocess.run stand-in answering the probe's own git add of the bad name with `add`.""" + real_run = subprocess.run + + def run(cmd: list[str], *args: Any, **kwargs: Any) -> Any: + if control is not None and cmd == ["git", "add", "--", "control.txt"]: + return control(cmd) + if cmd[:3] == ["git", "add", "--"] and kwargs.get("cwd") and cmd[3].startswith("bad-"): + return add(cmd) + if commit is not None and cmd[:2] == ["git", "commit"] and "add invalid-utf8 name" in cmd: + return commit(cmd) + return real_run(cmd, *args, **kwargs) + + return run + + +def _fail(cmd: list[str]) -> Any: + raise subprocess.CalledProcessError(128, cmd) + + +class HubTrackedProbeCase(unittest.TestCase): + """The invalid-byte probe skips only where git on Windows refuses the name, and fails loudly otherwise.""" + + def probe(self, run: Any, windows: bool) -> BaseException: + case = HubTrackedCase("test_non_utf8_filename_round_trips") + self.addCleanup(case.doCleanups) + with ( + mock.patch.object(audit, "_IS_WINDOWS", windows), + mock.patch.object(subprocess, "run", run), + ): + try: + case.test_non_utf8_filename_round_trips() + except (unittest.SkipTest, subprocess.CalledProcessError) as error: + if "host cannot create it" in str(error) or "no candidate byte" in str(error): + self.skipTest("this host cannot exercise the invalid-byte probe") + return error + self.fail("the probe neither skipped nor failed") + + def test_a_failing_add_of_the_probe_file_skips_on_windows(self) -> None: + error = self.probe(_probe_run(_fail), windows=True) + self.assertIsInstance(error, unittest.SkipTest) + self.assertIn("git cannot add it", str(error)) + + def test_a_failing_control_add_fails_loudly_on_windows(self) -> None: + ok = _probe_run(lambda cmd: subprocess.CompletedProcess(cmd, 0), control=_fail) + error = self.probe(ok, windows=True) + assert isinstance(error, subprocess.CalledProcessError) + self.assertEqual(error.cmd, ["git", "add", "--", "control.txt"]) + + def test_a_failing_add_of_the_probe_file_fails_loudly_elsewhere(self) -> None: + error = self.probe(_probe_run(_fail), windows=False) + assert isinstance(error, subprocess.CalledProcessError) + self.assertEqual(error.cmd[:3], ["git", "add", "--"]) + + def test_another_git_failure_still_fails_loudly(self) -> None: + run = _probe_run(lambda cmd: subprocess.CompletedProcess(cmd, 0), commit=_fail) + error = self.probe(run, windows=True) + assert isinstance(error, subprocess.CalledProcessError) + self.assertEqual(error.cmd[:2], ["git", "commit"]) + + +GIT_IDENTITY = ( + ["git", "init", "-q"], + ["git", "config", "user.email", "test@test.invalid"], + ["git", "config", "user.name", "test"], + ["git", "config", "commit.gpgsign", "false"], +) + + +def clear_history_caches() -> None: + audit._require_full_history.cache_clear() + audit._git_revisions.cache_clear() + audit.git_blob_in_file_history.cache_clear() + + +class SectionCase(unittest.TestCase): + """Holds region splitting, section extraction and stripping, and fence handling.""" + + def test_forked_release_region_hashes_differently(self) -> None: + region = audit.split_jobs(REL_OK).get("github-release") + forked_region = audit.split_jobs( + REL_OK.replace(" merge-multiple: true\n", " artifact-ids: 1\n") + ).get("github-release") + self.assertIsNotNone( + region, "verbatim: forked github-release region should hash differently" + ) + self.assertIsNotNone( + forked_region, "verbatim: forked github-release region should hash differently" + ) + self.assertNotEqual( + audit.content_hash(region), + audit.content_hash(forked_region), + "verbatim: forked github-release region should hash differently", + ) + + def test_extract_section_region_and_hash(self) -> None: + md = "# Title\n\n## Alpha\n\nbody a\n\n```\n## not a heading\n```\n\n### nested\nstill alpha\n\n## Beta\n\nbody b\n" + a, b, gone = ( + audit.extract_section(md, "Alpha"), + audit.extract_section(md, "Beta"), + audit.extract_section(md, "Gamma"), + ) + spaced = audit.extract_section("## Alpha\n\nbody a\n", "Alpha") + msg = "section: extract_section region/hash behavior" + self.assertIsNotNone(a, msg) + self.assertTrue(a.startswith("## Alpha"), msg) + self.assertIn("body a", a, msg) + self.assertIn("## not a heading", a, msg) + self.assertIn("still alpha", a, msg) + self.assertNotIn("body b", a, msg) + self.assertIsNotNone(b, msg) + self.assertTrue(b.startswith("## Beta"), msg) + self.assertIn("body b", b, msg) + self.assertNotIn("body a", b, msg) + self.assertIsNone(gone, msg) + self.assertIsNotNone(spaced, msg) + self.assertTrue(spaced.startswith("## Alpha"), msg) + self.assertNotEqual( + audit.content_hash(a), + audit.content_hash(audit.extract_section(md.replace("## Alpha", "## alpha"), "Alpha")), + msg, + ) + + def test_mismatched_marker_in_fence_is_not_a_boundary(self) -> None: + nested = "## Alpha\n```md\n~~~\n## Fake\n```\nreal content\n" + nested_extract = audit.extract_section(nested, "Alpha") + nested_strip = audit.strip_sections( + "## Keep\ntext\n```md\n~~~\n## Fake Drop\n```\nmore text\n## Drop\nreal drop content\n", + ["Drop"], + ) + msg = "section: a mismatched marker nested in a fence ends the region early" + self.assertIsNotNone(nested_extract, msg) + self.assertIn("## Fake", nested_extract, msg) + self.assertIn("real content", nested_extract, msg) + self.assertIn("## Fake Drop", nested_strip, msg) + self.assertNotIn("real drop content", nested_strip, msg) + self.assertIn("more text", nested_strip, msg) + + def test_unfenced_text(self) -> None: + cases = [ + ( + "a simple ``` fence excludes its content", + "```\n## Phantom\n```\n## Real\n", + "## Real\n", + ), + ( + "a simple ~~~ fence excludes its content", + "~~~\n## Phantom\n~~~\n## Real\n", + "## Real\n", + ), + ( + "a ~~~ line nested inside a ``` fence does not close it", + "```md\n~~~\n## Phantom\n```\n## Real\n", + "## Real\n", + ), + ( + "a shorter ``` cannot close a longer ```` fence", + "````md\n## Phantom\n```\n## Real\n````\n", + "", + ), + ( + "a longer closing fence than the opener still closes", + "```\n## Phantom\n````\n## Real\n", + "## Real\n", + ), + ( + "a marker run followed by trailing text does not close", + "```\n## Phantom\n```not-a-fence\n## Real\n```\n## Real2\n", + "## Real2\n", + ), + ( + "trailing whitespace after the marker run still closes", + "```\n## Phantom\n``` \n## Real\n", + "## Real\n", + ), + ( + "a 3-space-indented closing fence still closes", + "```\n## Phantom\n ```\n## Real\n", + "## Real\n", + ), + ( + "a 4-space-indented closing fence remains content, not a boundary", + "```\n## Phantom\n ```\n## Still Phantom\n```\n## Real\n", + "## Real\n", + ), + ( + "a 4-space-indented opener never opens a fence", + " ```\n## Real\n", + " ```\n## Real\n", + ), + ( + "a backtick in a backtick-fence info string is not a valid opener", + "```md`\n## Real\n", + "```md`\n## Real\n", + ), + ( + "a backtick in a tilde-fence info string does not disqualify it", + "~~~md`\n## Phantom\n~~~\n## Real\n", + "## Real\n", + ), + ] + for label, doc, want in cases: + with self.subTest(label): + self.assertEqual(audit.unfenced_text(doc), want) + + def test_undeclared_h2_headings(self) -> None: + cases = [ + ( + "a declared H2 is not flagged", + "# AGENTS\n\n## Fleet Bootstrap\n\nText.\n", + {"fleet bootstrap"}, + [], + ), + ( + "an undeclared H2 is flagged", + "# AGENTS\n\n## Fleet Bootstrap\n\nText.\n\n## Local Notes\n\nRepo-specific.\n", + {"fleet bootstrap"}, + ["local notes"], + ), + ( + "declared-name match is case-insensitive", + "# AGENTS\n\n## fleet BOOTSTRAP\n\nText.\n", + {"fleet bootstrap"}, + [], + ), + ( + "an un-normalized declared set (mixed case, untrimmed) is normalized here, not trusted", + "# AGENTS\n\n## Fleet Bootstrap\n\nText.\n", + {" Fleet Bootstrap "}, + [], + ), + ( + "an H1 title and a nested H3 are not judged, only H2", + "# Local Notes\n\n## Fleet Bootstrap\n\n### Local Notes\n\nText.\n", + {"fleet bootstrap"}, + [], + ), + ( + "a fenced sample showing heading syntax is not a real heading", + "# AGENTS\n\n## Fleet Bootstrap\n\n```\n## Local Notes\n```\n", + {"fleet bootstrap"}, + [], + ), + ( + "a backtick in a backtick-fence info string is not a valid opener, so the heading after it is real", + "# AGENTS\n\n## Fleet Bootstrap\n\n```md`\n## Local Notes\n", + {"fleet bootstrap"}, + ["local notes"], + ), + ( + "a repo's own local content, undeclared, is flagged even in a file with its own declared sections", + ( + "# Copilot Instructions\n\n## GitHub Copilot Review Runbook\n\nText.\n\n" + "## Development Workflow\n\nLocal build and test steps.\n\n" + "## Command Line Usage\n\nLocal CLI reference.\n" + ), + {"github copilot review runbook"}, + ["command line usage", "development workflow"], + ), + ] + for label, doc, declared, want in cases: + with self.subTest(label): + self.assertEqual(audit.undeclared_h2_headings(doc, declared), want) + + +class ScanCase(unittest.TestCase): + """Holds the template-reference and version-literal scans outside verbatim sections.""" + + def test_template_ref_outside_verbatim(self) -> None: + boot = "## Fleet Bootstrap\n\nThe canonical rules live in `github.com/acme/Hub`.\n" + owned = "## Where the Rules Live\n\nReport a rule discrepancy to acme/Hub.\n" + clean_doc = ( + "# AGENTS\n\n" + + boot + + "\n## Where the Rules Live\n\nState the behavior, not the destination.\n" + ) + dirty_doc = "# AGENTS\n\n" + boot + "\n" + owned + cases = [ + ("hub name only inside the verbatim section", clean_doc, {"Fleet Bootstrap"}, False), + ("hub name in prose the repo owns", dirty_doc, {"Fleet Bootstrap"}, True), + ("same document with nothing declared verbatim still flags", clean_doc, set(), True), + ( + "CRLF document excises the same way", + clean_doc.replace("\n", "\r\n"), + {"Fleet Bootstrap"}, + False, + ), + ( + "a re-cased verbatim heading still excises", + clean_doc.replace("## Fleet Bootstrap", "## fleet bootstrap"), + {"Fleet Bootstrap"}, + False, + ), + ( + "no hub reference at all", + "# AGENTS\n\n## Where the Rules Live\n\nNothing to see.\n", + {"Fleet Bootstrap"}, + False, + ), + ( + "the heading appearing twice excises both", + "# AGENTS\n\n" + boot + "\n## Notes\n\n" + boot, + {"Fleet Bootstrap"}, + False, + ), + ( + "a fenced copy of the section is prose, not the section", + "# AGENTS\n\n## Notes\n\n```\n" + boot + "```\n\n" + boot, + {"Fleet Bootstrap"}, + True, + ), + ] + for label, doc, verb, want in cases: + with self.subTest(label): + self.assertEqual(audit.template_ref_outside_verbatim(doc, verb, "acme/Hub"), want) + + def test_version_literals_outside_verbatim(self) -> None: + sha = "0123456789abcdef0123456789abcdef01234567" + cases = [ + ( + "a release number is flagged", + "Pinned at hub release `2.0.657`.\n", + set(), + ["2.0.657"], + ), + ("a full SHA is flagged", f"uses: o/r/x.yml@{sha} # 2.0.1\n", set(), ["2.0.1", sha]), + ("an uppercase SHA is flagged", f"Pinned at {sha.upper()}.\n", set(), [sha.upper()]), + ("an abbreviated SHA is flagged", "Pinned at `f3b4cc9`.\n", set(), ["f3b4cc9"]), + ( + "an uppercase abbreviated SHA is flagged", + "Pinned at `F3B4CC9`.\n", + set(), + ["F3B4CC9"], + ), + ("an all-letter hex word is not a SHA", "A facade over deadbeef.\n", set(), []), + ("a hex color is not a SHA", "Color #1f2937ff.\n", set(), []), + ("a hash-prefixed hex token is not a SHA", "See #f3b4cc9.\n", set(), []), + ( + "an identifier suffix is not a SHA", + "Name foo_1a2b3c4d and 123e4567-e89b.\n", + set(), + [], + ), + ("an image tag SHA is flagged", "Image app:sha-1a2b3c4.\n", set(), ["1a2b3c4"]), + ("a hex constant is not a SHA", "Mask 0x7fffffff.\n", set(), []), + ( + "a worked example is flagged", + "1.0.12 publishes as 1.0.13.\n", + set(), + ["1.0.12", "1.0.13"], + ), + ("a placeholder example is clean", "`1.0.N` publishes as `1.0.(N+1)`.\n", set(), []), + ("the mechanism alone is clean", "SHA-pinned to a hub release.\n", set(), []), + ("a dotted quad is not a version", "Host 192.168.1.10 serves it.\n", set(), []), + ("a two-part version is not a pin", "Python 3.13 and 3.14.\n", set(), []), + ( + "a literal inside a verbatim section is the hub's", + "# W\n\n## Locked\n\nPinned at 2.0.1.\n\n## Mine\n\nClean.\n", + {"Locked"}, + [], + ), + ( + "a literal outside the verbatim section is flagged", + "# W\n\n## Locked\n\nClean.\n\n## Mine\n\nPinned at 2.0.2.\n", + {"Locked"}, + ["2.0.2"], + ), + ] + for label, doc, verb, want in cases: + with self.subTest(label): + self.assertEqual(audit.version_literals_outside_verbatim(doc, verb), sorted(want)) + + +class RenderCase(unittest.TestCase): + """Holds the issue generator, the README mirror check, link stripping, and cspell detection.""" + + def test_render_issue_grouping(self) -> None: + fe = {"name": "Widget", "types": ["python"]} + it, ib = audit.render_issue( + fe, + [ + ("LETTER", "file: X absent"), + ("DRIFT", "verbatim: Y differs"), + ("ERROR", "gh failed"), + ], + "main", + "abc1234", + "2026-01-01T00:00:00Z", + "hub123", + ) + et, eb = audit.render_issue(fe, [], "main", "abc1234", "2026-01-01T00:00:00Z", "hub123") + msg = "issue: render_issue grouping/title/empty behavior" + self.assertIn("Widget", it, msg) + self.assertIn("(3 findings)", it, msg) + self.assertIn("## Must fix", ib, msg) + self.assertIn("**LETTER** file: X absent", ib, msg) + self.assertIn("## Converge", ib, msg) + self.assertIn("verbatim: Y differs", ib, msg) + self.assertIn("## Could not verify", ib, msg) + self.assertIn("(0 findings)", et, msg) + self.assertIn("nothing to converge", eb, msg) + + def test_title_and_intro_mirror(self) -> None: + r_md = "# Widget \n\nDoes widget things.\n\n## Build\n" + h_md = "# Widget\n\nDoes widget things.\n\n## Release History\n" + h_bad = "# Widget\n\nDoes other things.\n\n## Release History\n" + r_two = "# W\n\nLine one.\n\nLine two.\n\n## Build\n" + h_joined = "# W\n\nLine one.\nLine two.\n\n## Release History\n" + msg = "history: README/HISTORY title+intro mirror detection" + self.assertEqual(audit.title_and_intro(r_md), audit.title_and_intro(h_md), msg) + self.assertNotEqual(audit.title_and_intro(r_md), audit.title_and_intro(h_bad), msg) + self.assertNotEqual(audit.title_and_intro(r_two), audit.title_and_intro(h_joined), msg) + + def test_strip_md_links(self) -> None: + linked = "Utility to clean [media](https://x.example/Foo_(bar)) per the [spec][spec-ref]." + nested = "See [API [docs]](https://example.test/a_(b)_(c)) for details." + inner_link = "See [[docs](url)] for details." + escaped_bracket = r"See [API \[docs](url) for details." + msg = "description: strip_md_links behavior" + self.assertEqual(audit.strip_md_links(linked), "Utility to clean media per the spec.", msg) + self.assertEqual(audit.strip_md_links("Plain intro line."), "Plain intro line.", msg) + self.assertEqual(audit.strip_md_links(nested), "See API [docs] for details.", msg) + self.assertEqual(audit.strip_md_links(inner_link), "See [docs] for details.", msg) + self.assertEqual(audit.strip_md_links(escaped_bracket), r"See API \[docs for details.", msg) + + def test_strip_md_links_stays_linear(self) -> None: + pathological = "[" * 20000 + start = time.monotonic() + audit.strip_md_links(pathological) + elapsed = time.monotonic() - start + self.assertLessEqual( + elapsed, + 1.0, + f"description: strip_md_links took {elapsed:.2f}s on unmatched brackets, expected linear", + ) + + def test_workspace_cspell_words(self) -> None: + ws_dup = '{ "settings": { "cSpell.words": ["foo"] } }' + ws_ok = '{ "settings": { "editor.rulers": [100] }, "note": "words live in cspell.json" }' + msg = "cspell: workspace word-list detection" + self.assertTrue(audit.workspace_cspell_words(ws_dup), msg) + self.assertFalse(audit.workspace_cspell_words(ws_ok), msg) + + +class BranchCase(unittest.TestCase): + """Holds branch drift classification and ground-truth branch resolution.""" + + def test_classify_branch_drift(self) -> None: + bd_base = {"keep": "a", "moda": "1", "modb": "2", "deld": "e", "devonly": "x"} + bd_main = { + "keep": "a", + "moda": "9", + "modb": "9", + "add": "n", + "devonly": "x", + } + bd_dev = { + "keep": "a", + "moda": "1", + "modb": "7", + "add": "m", + "deld": "e", + "devonly": "y", + } + bd_behind, bd_diverged = audit.classify_branch_drift(bd_base, bd_main, bd_dev) + self.assertEqual( + (bd_behind, bd_diverged), + (["deld", "moda"], ["add", "modb"]), + f"branch-drift classify -> behind={bd_behind} diverged={bd_diverged}", + ) + + def test_ground_branch_of(self) -> None: + cases = [ + ({}, None, None, "main"), + ({"groundTruthBranch": "develop"}, None, None, "develop"), + ({"groundTruthBranch": "main"}, "develop", None, "develop"), + ({}, "feature/x", None, "feature/x"), + ({}, None, {"groundTruthBranch": "develop"}, "develop"), + ({"groundTruthBranch": "main"}, None, {"groundTruthBranch": "develop"}, "main"), + ({"groundTruthBranch": ""}, None, {"groundTruthBranch": "develop"}, ""), + ({}, None, {"groundTruthBranch": ""}, ""), + ] + for entry, branch, defaults, want in cases: + with self.subTest(f"{entry} + {branch} + {defaults}"): + self.assertEqual(audit.ground_branch_of(entry, branch, defaults), want) + + +class ShallowCloneCase(unittest.TestCase): + """Holds the shallow-clone refusal on the history reads and on the public paths.""" + + def make_dir(self) -> pathlib.Path: + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + return pathlib.Path(tmp.name) + + def setUp(self) -> None: + clear_history_caches() + self.addCleanup(clear_history_caches) + + def test_history_reads_refuse_shallow_clone(self) -> None: + tmp_path = self.make_dir() + full = tmp_path / "full" + full.mkdir() + for cmd in ( + *GIT_IDENTITY, + ["git", "commit", "-q", "--allow-empty", "-m", "one"], + ["git", "commit", "-q", "--allow-empty", "-m", "two"], + ): + subprocess.run(cmd, cwd=full, check=True, capture_output=True) + shallow = tmp_path / "shallow" + subprocess.run( + ["git", "clone", "-q", "--depth", "1", full.as_uri(), str(shallow)], + check=True, + capture_output=True, + ) + for label, root, want_refuse in (("shallow", shallow, True), ("full", full, False)): + with mock.patch.object(audit, "ROOT", root): + clear_history_caches() + reads = ( + ("revisions", lambda: audit._git_revisions("absent.txt", rev="HEAD")), + ( + "blob", + lambda: audit.git_blob_in_file_history("absent.txt", "0" * 40, rev="HEAD"), + ), + ) + for read, call in reads: + with self.subTest(f"{label} clone, {read}"): + try: + call() + msg = None + except audit.ShallowCloneError as e: + msg = str(e) + good = (msg is not None) == want_refuse + if want_refuse: + good = ( + good + and "Run `git fetch --unshallow origin` in that checkout." + in (msg or "") + ) + self.assertTrue(good, f"shallow-clone guard: {label} clone, {read}") + + def test_public_paths_refuse_shallow_clone(self) -> None: + tmp_path = self.make_dir() + hub_dir = tmp_path / "hub" + hub_dir.mkdir() + rels = ( + "registry/repos.json", + "repo-config/settings.json", + "spec/secrets.json", + "spec/files.json", + "spec/project-types.json", + "spec/readme-sections.json", + "spec/third-party-tools.json", + "spec/divergences.json", + ) + for rel in rels: + (hub_dir / rel).parent.mkdir(parents=True, exist_ok=True) + (hub_dir / rel).write_bytes((audit.ROOT / rel).read_bytes()) + for cmd in ( + *GIT_IDENTITY, + ["git", "add", "."], + ["git", "commit", "-q", "-m", "one"], + ["git", "commit", "-q", "--allow-empty", "-m", "two"], + ): + subprocess.run(cmd, cwd=hub_dir, check=True, capture_output=True) + shallow_hub = tmp_path / "shallow-hub" + subprocess.run( + ["git", "clone", "-q", "--depth", "1", hub_dir.as_uri(), str(shallow_hub)], + check=True, + capture_output=True, + ) + name = next( + r["name"] + for r in audit.load("registry/repos.json")["repos"] + if r.get("status") == "cataloged" + ) + + def history_read_only(entry, spec, branch=None): + audit._git_revisions("absent.txt", rev="HEAD") + return [], "" + + with ( + mock.patch.object(audit, "ROOT", shallow_hub), + mock.patch.object(audit, "audit_repo", history_read_only), + ): + for flag_argv, label, want_code in ( + ([name], "sweep", 1), + (["--issue", name], "issue", 2), + ): + with self.subTest(label): + audit._require_full_history.cache_clear() + audit._git_revisions.cache_clear() + out, err = io.StringIO(), io.StringIO() + try: + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = audit.main(flag_argv) + except Exception as e: # noqa: BLE001 + code = f"raised {type(e).__name__}" + text = out.getvalue() if label == "sweep" else err.getvalue() + good = ( + code == want_code + and "Run `git fetch --unshallow origin` in that checkout." in text + and (label == "sweep" or not out.getvalue()) + ) + if label == "sweep": + good = good and "ERROR" in text + self.assertTrue( + good, f"shallow-clone refusal, public {label} path (code {code})" + ) + + +class ParseArgsCase(unittest.TestCase): + """Holds command line parsing, where a repo name and a flag value stay distinct.""" + + def test_parse_args(self) -> None: + cases = [ + ([], [], None, False, False), + (["Utilities"], ["Utilities"], None, False, False), + (["Utilities", "--branch", "develop"], ["Utilities"], "develop", False, False), + (["--branch=feature/x", "Utilities"], ["Utilities"], "feature/x", False, False), + (["--issue", "Utilities"], ["Utilities"], None, True, False), + (["--selftest"], [], None, False, True), + ] + for argv, names, branch, issue, selftest in cases: + with self.subTest(str(argv or ["(no args)"])): + got = audit.parse_args(argv) + self.assertEqual( + (got.names, got.branch, got.issue, got.selftest), + (names, branch, issue, selftest), + ) + + +NOTE_SPEC = { + "types": { + "types": { + "hugo": {"checks": [{"id": "hugo.generator.pinned"}]}, + "docker": {"checks": [{"id": "docker.cache.registry"}]}, + }, + "crossCutting": {"setup": {"checks": [{"id": "setup.driftnotes.current"}]}}, + } +} + +NOTE_CASES = [ + # The end-of-sentence period is the case an end-anchored pattern misses, and it is how both notes this form was introduced for are written. + ( + "check id followed by a period", + ["hugo"], + ["Pinned in two workflows (hugo.generator.pinned)."], + 0, + ["does not evaluate"], + ), + ( + "check id mid-sentence", + ["hugo"], + ["The pin (hugo.generator.pinned) is duplicated."], + 0, + ["does not evaluate"], + ), + ( + "cross-cutting id, which no repo declares", + ["hugo"], + ["Notes go stale (setup.driftnotes.current)."], + 0, + ["does not evaluate"], + ), + ( + "check id of an undeclared type", + ["hugo"], + ["Cache layer (docker.cache.registry)."], + 0, + ["this repo does not declare"], + ), + ( + "a malformed types value does not abort the note check", + None, + ["Vendored helper, not tracked (python.directories.declared)."], + 0, + [], + ), + ( + "a detection suppression on a repo without the type", + ["hugo"], + ["Vendored helper, not tracked (python.directories.declared)."], + 0, + [], + ), + ( + "check id absent from the catalog", + ["hugo"], + ["Theme record (hugo.vendored.provenence)."], + 0, + ["does not define"], + ), + # A version string in parentheses is the false positive the id shape has to exclude. + ( + "parenthesized version, not a check id", + ["hugo"], + ["Generator held at (0.164.0) by the composite action."], + 0, + [], + ), + ( + "permanent deviation, no marker and no id", + ["hugo"], + ["Relies on validate-task, having no get-version-task."], + 0, + [], + ), + ( + "marker note on a clean audit", + ["hugo"], + ["The sibling doc is pending fleet ratification."], + 0, + ["but the audit is clean"], + ), + # The case the old gate suppressed outright: one unclearable finding, and the note never checked. + ( + "marker note with findings open", + ["hugo"], + ["The sibling doc is pending fleet ratification."], + 1, + ["while 1 finding(s) are open"], + ), +] + +CONFORMANT = ( + "# Fixture\n\nA fixture repository.\n\n## Build and Distribution\n\n- **Source Code**: [GitHub][gh]\n\n" + "### Build Status\n\n[![Release Status][a]][x]\\\n[![Last Commit][b]][x]\n\n" + "### Releases\n\n[![GitHub Release][c]][x]\\\n[![GitHub Pre-Release][d]][x]\n\n" + "### Release Notes\n\n**Version**: 1.0\n\n## Table of Contents\n\n- [Overview](#overview)\n\n" + "## Overview\n\nWhat it does.\n\n## Whatever This Repo Calls It\n\nRepo-specific.\n\n" + "## Questions or Issues\n\nOpen an issue.\n\n## 3rd Party Tools\n\n- [Thing][t]\n\n" + "## License\n\nLicensed under the [MIT License][license]\\\n![License][license-shield]\n\n" + "\n\n" + "[a]: https://img.shields.io/github/actions/workflow/status/o/r/publish-release.yml?label=Releases%20Build\n" + "[b]: https://img.shields.io/github/last-commit/o/r\n" + "[c]: https://img.shields.io/github/v/release/o/r?label=GitHub%20Release\n" + "[d]: https://img.shields.io/github/v/release/o/r?include_prereleases&filter=*-g*&label=GitHub%20Pre-Release\n" + "[license-shield]: https://img.shields.io/github/license/o/r\n" +) + +# The four repos that suffix every heading for the ToC extension must read identically to the plain form. +OMIT_TOC = re.sub(r"^(#{2,3} .*)$", r"\1 ", CONFORMANT, flags=re.MULTILINE) + +README_CASES = [ + ("a conformant README, repo-specific section included", CONFORMANT, set(), True, 0), + ( + "a second intro paragraph is not a finding", + CONFORMANT.replace( + "A fixture repository.\n", + "A fixture repository.\n\nAnd a clarifying paragraph about it.\n", + ), + set(), + True, + 0, + ), + ( + "a retired section name reports a rename", + CONFORMANT.replace("## Overview", "## Features"), + set(), + True, + 1, + ), + ( + "Questions or Issues is optional while private", + CONFORMANT.replace("## Questions or Issues\n\nOpen an issue.\n\n", ""), + set(), + False, + 0, + ), + ( + "Questions or Issues is required when public", + CONFORMANT.replace("## Questions or Issues\n\nOpen an issue.\n\n", ""), + set(), + True, + 1, + ), + ( + "3rd Party Tools is required in every repo", + CONFORMANT.replace("## 3rd Party Tools\n\n- [Thing][t]\n\n", ""), + set(), + True, + 1, + ), + ( + "a Table of Contents is required with no size threshold", + CONFORMANT.replace("## Table of Contents\n\n- [Overview](#overview)\n\n", ""), + set(), + True, + 1, + ), + ( + "an out-of-order declared section reports once", + CONFORMANT.replace("## Questions or Issues\n\nOpen an issue.\n\n", "").replace( + "## Overview", "## Questions or Issues\n\nOpen an issue.\n\n## Overview" + ), + set(), + True, + 1, + ), + ( + "License must be the last section", + CONFORMANT + "\n## TODO\n\nA backlog.\n", + set(), + True, + 1, + ), + ( + "a missing Release Notes sub-section reports once", + CONFORMANT.replace("### Release Notes\n\n**Version**: 1.0\n\n", ""), + set(), + True, + 1, + ), + ( + "a `## ` line inside a fence is not a section", + CONFORMANT.replace("What it does.", "```md\n## Not A Section\n```"), + set(), + True, + 0, + ), + ("Usage and Installation are N/A for source-only", CONFORMANT, {"source-only"}, True, 0), + ("a ToC-omit comment on every heading changes nothing", OMIT_TOC, set(), True, 0), +] + +DOCKER_SHIELD = "[![Docker Latest][e]][x]\n[e]: https://img.shields.io/docker/v/o/r/latest\n" + +INLINE_ALL = re.sub( + r"!\[([^\]]*)\]\[([a-z-]+)\]", + lambda m: f"![{m.group(1)}](https://img.shields.io/PLACEHOLDER-{m.group(2)})", + CONFORMANT, +) + +SHIELD_CASES = [ + ("base shields only, a repo publishing nothing", CONFORMANT, {}, 0), + ("a ToC-omit comment on every heading changes nothing", OMIT_TOC, {}, 0), + ("a docker repo owes a version shield", CONFORMANT, {"publish": [{"target": "docker"}]}, 1), + ( + "a docker repo carrying one is satisfied, whatever its channel", + CONFORMANT.replace( + "[![GitHub Pre-Release][d]][x]\n", + "[![GitHub Pre-Release][d]][x]\\\n" + DOCKER_SHIELD, + ), + {"publish": [{"target": "docker"}]}, + 0, + ), + ( + "a nuget repo owes one version shield, no prerelease", + CONFORMANT, + {"publish": [{"target": "nuget"}]}, + 1, + ), + ( + "a pypi repo owes one version shield, no prerelease", + CONFORMANT, + {"publish": [{"target": "pypi"}]}, + 1, + ), + ( + "a caption the repo spells differently is not a finding", + CONFORMANT.replace("![Release Status]", "![Lint Build]"), + {}, + 0, + ), + ( + "an extra shield beyond the class is not a finding", + CONFORMANT.replace( + "[![Last Commit][b]][x]", "[![Last Commit][b]][x]\\\n[![Last Build][a]][x]" + ), + {}, + 0, + ), + ( + "a shield in the wrong sub-section does not count as present", + CONFORMANT.replace("[![GitHub Release][c]][x]\\\n", ""), + {}, + 1, + ), + # A badge shown as a code sample is markup, so it satisfies nothing and trips nothing. + # Every region the README checks read comes through readme_region, which strips fences at the root. + ( + "a fenced badge sample does not satisfy a required shield", + CONFORMANT.replace( + "[![GitHub Release][c]][x]\\\n", "```md\n[![GitHub Release][c]][x]\n```\n" + ), + {}, + 1, + ), + ( + "a fenced license shield does not trip the exclusive rule", + CONFORMANT.replace("## Overview", "```md\n![License][license-shield]\n```\n\n## Overview"), + {}, + 0, + ), + ( + "a retired badge service is reported wherever it sits", + CONFORMANT.replace( + "[license-shield]: https://img.shields.io/github/license/o/r\n", + "[license-shield]: https://img.shields.io/github/license/o/r\n[last-build-shield]: https://byob.yarr.is/o/r/lastbuild\n", + ), + {}, + 1, + ), + # The case above named every placement and read only the reference block, so an inline badge was invisible rather than wrong. + # That is the reading shield_endpoints already takes for every other shield, and this one had not taken it. + ( + "a retired badge written inline is reported", + CONFORMANT.replace( + "[![Last Commit][b]][x]", + "[![Last Commit][b]][x]\\\n![Last Build](https://byob.yarr.is/o/r/lastbuild)", + ), + {}, + 1, + ), + ( + "a retired badge defined and rendered is one finding, not two", + CONFORMANT.replace( + "[![Last Commit][b]][x]", + "[![Last Commit][b]][x]\\\n![Last Build][last-build-shield]", + ).replace( + "[license-shield]: https://img.shields.io/github/license/o/r\n", + "[license-shield]: https://img.shields.io/github/license/o/r\n[last-build-shield]: https://byob.yarr.is/o/r/lastbuild\n", + ), + {}, + 1, + ), + ( + "a retired badge shown as a fenced sample is markup", + CONFORMANT.replace( + "## Overview", + "```md\n![Last Build](https://byob.yarr.is/o/r/lastbuild)\n```\n\n## Overview", + ), + {}, + 0, + ), + # The wording follows which of the three shapes it is, since a definition nothing renders is not rendering anything. + # Saying it renders sends the reader looking for a badge that is not on the page. + ( + "an unrendered definition says so rather than claiming a render", + CONFORMANT.replace( + "[license-shield]: https://img.shields.io/github/license/o/r\n", + "[license-shield]: https://img.shields.io/github/license/o/r\n[last-build-shield]: https://byob.yarr.is/o/r/lastbuild\n", + ), + {}, + 1, + "nothing renders it", + ), + ( + "a rendered definition says renders", + CONFORMANT.replace( + "[![Last Commit][b]][x]", + "[![Last Commit][b]][x]\\\n![Last Build][last-build-shield]", + ).replace( + "[license-shield]: https://img.shields.io/github/license/o/r\n", + "[license-shield]: https://img.shields.io/github/license/o/r\n[last-build-shield]: https://byob.yarr.is/o/r/lastbuild\n", + ), + {}, + 1, + "renders the byob", + ), + # The same endpoint rendered inline leaves this definition unused, so attributing by URL would credit the render to a reference nothing uses. + ( + "an unused definition beside an inline render is not credited with it", + CONFORMANT.replace( + "[![Last Commit][b]][x]", + "[![Last Commit][b]][x]\\\n![Last Build](https://byob.yarr.is/o/r/lastbuild)", + ).replace( + "[license-shield]: https://img.shields.io/github/license/o/r\n", + "[license-shield]: https://img.shields.io/github/license/o/r\n[last-build-shield]: https://byob.yarr.is/o/r/lastbuild\n", + ), + {}, + 1, + "rendered elsewhere", + ), + ( + "the pre-release shield is told from the release shield by its query", + CONFORMANT.replace( + "?include_prereleases&filter=*-g*&label=GitHub%20Pre-Release", + "?label=Another%20Release", + ), + {}, + 1, + ), + ( + "a pre-release shield without the prerelease filter is a finding", + CONFORMANT.replace("&filter=*-g*&", "&"), + {}, + 1, + "with `filter=*-g*`", + ), + ( + "a missing release shield names the query it forbids", + CONFORMANT.replace("[![GitHub Release][c]][x]\\\n", ""), + {}, + 1, + "without `include_prereleases`", + ), + # The license shield is an ordinary member of the base class, addressed to a different section. + ("the license shield in the closing License section", CONFORMANT, {}, 0), + ( + "no license shield at all", + CONFORMANT.replace("\\\n![License][license-shield]", ""), + {}, + 1, + ), + ( + "the license shield left under Releases", + CONFORMANT.replace("\\\n![License][license-shield]", "").replace( + "[![GitHub Pre-Release][d]][x]", + "[![GitHub Pre-Release][d]][x]\\\n![License][license-shield]", + ), + {}, + 2, + ), + ( + "a shield's own reference definition is not a use of it", + CONFORMANT.replace("![License][license-shield]\n", ""), + {}, + 1, + ), + # An inline shield is resolved rather than silently invisible. + # Reading references alone let a repo writing every badge inline pass by carrying nothing the check could see. + ( + "inline shields count as present", + INLINE_ALL.replace("PLACEHOLDER-a", "github/actions/workflow/status/o/r") + .replace("PLACEHOLDER-b", "github/last-commit/o/r") + .replace("PLACEHOLDER-c", "github/v/release/o/r") + .replace("PLACEHOLDER-d", "github/v/release/o/r?include_prereleases&filter=*-g*") + .replace("PLACEHOLDER-license-shield", "github/license/o/r"), + {}, + 0, + ), + ( + "an inline shield in the wrong section is still exclusive", + CONFORMANT.replace( + "## Overview", + "![License](https://img.shields.io/github/license/o/r)\n\n## Overview", + ), + {}, + 1, + ), +] + +LINKS_OK = ( + "# F\n\nA fixture.\n\n## X\n\n[a](#x) [b][gh] [c][docker-hub-link] [d][agents] [e][upstream-link]\n\n" + "![License][license-shield]\n\n" + "\n\n[x-anchor]: #x\n\n\n\n[license-shield]: https://img.shields.io/github/license/o/r\n\n" + "\n\n[actions-link]: https://github.com/o/r/actions\n[docker-hub-link]: https://hub.docker.com/r/o/r\n[github-link]: https://github.com/o/r\n\n" + "\n\n[agents]: ./AGENTS.md\n[license]: ./LICENSE\n\n" + "\n\n[upstream-link]: https://github.com/someone-else/their-repo\n" +) + +# Kept in reference-name order, so the case measures the per-target rule rather than tripping the sort check. +TWO_IMAGES = LINKS_OK.replace( + "[github-link]: https://github.com/o/r\n", + "[github-link]: https://github.com/o/r\n[lsio-docker-hub-link]: https://hub.docker.com/r/o/r-lsio\n", +) + +SWAPPED = LINKS_OK.replace( + "\n\n[x-anchor]: #x\n\n\n\n[license-shield]: https://img.shields.io/github/license/o/r\n", + "\n\n[license-shield]: https://img.shields.io/github/license/o/r\n\n\n\n[x-anchor]: #x\n", +) + +LINK_CASES = [ + ("a conformant reference block", LINKS_OK, 0, 0), + ( + "a shield reference not ending -shield", + LINKS_OK.replace("[license-shield]", "[licence]"), + 1, + 0, + ), + ( + "a URI reference not ending -link", + LINKS_OK.replace("[upstream-link]", "[upstream]"), + 1, + 0, + ), + ( + "a repo-local reference ending -link", + LINKS_OK.replace("[agents]:", "[agents-link]:"), + 1, + 0, + ), + ( + "the repo root named for the project", + LINKS_OK.replace("[github-link]", "[fixture-link]"), + 1, + 0, + ), + ("somebody else's GitHub repo is not renamed", LINKS_OK, 0, 0), + ("one Docker Hub image takes the bare name", LINKS_OK, 0, 0), + ("two Docker Hub images each take a target prefix", TWO_IMAGES, 1, 0), + ( + "two Docker Hub images, both prefixed, is clean", + TWO_IMAGES.replace( + "[docker-hub-link]: https://hub.docker.com/r/o/r\n", + "[base-docker-hub-link]: https://hub.docker.com/r/o/r\n", + ), + 0, + 0, + ), + ( + "an undeclared group header", + LINKS_OK.replace("", ""), + 0, + 1, + ), + ("groups out of the declared order", SWAPPED, 0, 1), + ( + "a group not sorted by reference name", + LINKS_OK.replace( + "[agents]: ./AGENTS.md\n[license]: ./LICENSE\n", + "[license]: ./LICENSE\n[agents]: ./AGENTS.md\n", + ), + 0, + 1, + ), + ( + "a reference in the wrong group", + LINKS_OK.replace("[license]: ./LICENSE\n", "").replace( + "\n", "\n\n[license]: ./LICENSE\n" + ), + 0, + 1, + ), + # A shields.io URL the document never renders is judged by that usage rather than by its host, so it is an ordinary URI. + # Across the fleet every one of the 119 img.shields.io definitions is rendered, so a host test decides nothing. + ( + "an unrendered shields.io reference is a URI, not a shield", + LINKS_OK.replace( + "[license-shield]: https://img.shields.io/github/license/o/r\n", + "[extra-shield]: https://img.shields.io/badge/never-rendered-blue\n[license-shield]: https://img.shields.io/github/license/o/r\n", + ), + 1, + 1, + ), + # A code sample showing badge markup is markup rather than a badge. + # Reading a fenced block would invent a group header, two definitions, and a rendered shield the document does not have. + ( + "a fenced code sample is not read as definitions", + LINKS_OK + + "\n## Sample\n\n```md\n\n\n[wrong-name]: https://example.test/\n![License][license-shield]\n```\n", + 0, + 0, + ), + # A dependency hosted where this project also publishes stays external, so it keeps its own name. + # The owner-scoped prefixes are what separate the two, and they live in the model rather than in the code. + ( + "somebody else's NuGet package is not renamed nuget-link", + LINKS_OK.replace( + "[upstream-link]: https://github.com/someone-else/their-repo", + "[upstream-link]: https://www.nuget.org/packages/Serilog/", + ), + 0, + 0, + ), + ( + "this project's own NuGet package is renamed", + LINKS_OK.replace( + "[upstream-link]: https://github.com/someone-else/their-repo", + "[upstream-link]: https://www.nuget.org/packages/o.Widget/", + ), + 1, + 1, + ), + # A comment that opens no group is not a group header. + # A markdownlint directive sits on its own line exactly like one, and reading it as a group reports a header the document never declared. + ( + "a directive comment is not a link group", + "\n" + LINKS_OK, + 0, + 0, + ), + # Also when it sits inside the reference block, where definitions do fall under it. + ( + "a directive inside the reference block is not a link group", + LINKS_OK.replace( + "[agents]: ./AGENTS.md", + "\n[agents]: ./AGENTS.md", + ), + 0, + 0, + ), + # A scheme makes a target a link rather than a file, so it is named `-link` and grouped External. + # Reading `mailto:` as a repo path would demand a bare name and the Repo group for a contact address. + ( + "a mailto target is a URI, not a repo path", + LINKS_OK.replace( + "[upstream-link]: https://github.com/someone-else/their-repo", + "[upstream-link]: mailto:someone@example.test", + ), + 0, + 0, + ), + ( + "a mailto target named as a repo path is reported", + LINKS_OK.replace( + "[upstream-link]: https://github.com/someone-else/their-repo", + "[upstream]: mailto:someone@example.test", + ), + 1, + 0, + ), + # Two names may point at one URL, and only the rendered one is a shield. + # Keying on the URL instead would make the plain link a shield because its twin is rendered. + ( + "a second reference to a rendered URL is not itself a shield", + LINKS_OK.replace( + "[upstream-link]: https://github.com/someone-else/their-repo", + "[upstream-link]: https://img.shields.io/github/license/o/r", + ), + 0, + 0, + ), +] + +TOOLS_HEAD = "# F\n\nA fixture.\n\n## 3rd Party Tools\n\n" +TOOLS_DEFS = "\n[cspell-link]: https://cspell.org\n[widget-link]: https://widget.example/\n" + +TOOL_CASES = [ + ( + "a catalog tool matching the catalog", + TOOLS_HEAD + + "| Tool | Role |\n| --- | --- |\n| [cspell][cspell-link] | Spell checker. |\n" + + TOOLS_DEFS, + 0, + ), + ( + "a catalog tool described differently", + TOOLS_HEAD + + "| Tool | Role |\n| --- | --- |\n| [cspell][cspell-link] | Spell-checks README.md in CI. |\n" + + TOOLS_DEFS, + 1, + ), + ( + "a catalog tool with no description at all", + TOOLS_HEAD + "- [cspell][cspell-link]\n" + TOOLS_DEFS, + 1, + ), + ( + "a catalog tool linked by another URL", + TOOLS_HEAD + + "| Tool | Role |\n| --- | --- |\n| [cspell][widget-link] | Spell checker. |\n" + + TOOLS_DEFS, + 1, + ), + ( + "a tool the catalog does not name is not judged", + TOOLS_HEAD + + "| Tool | Role |\n| --- | --- |\n| [Widget][widget-link] | whatever this repo calls it |\n" + + TOOLS_DEFS, + 0, + ), + ( + "the bullet form is read like the table form", + TOOLS_HEAD + "- [cspell][cspell-link] - Spell checker.\n" + TOOLS_DEFS, + 0, + ), + ( + "no section yields nothing, since its absence is already reported", + "# F\n\nA fixture.\n\n## Other\n\nx\n", + 0, + ), + # A definition inside a code sample is markup being shown, so it must not resolve a tool link. + # The sample sits after the real definition deliberately, so reading across fences would override it. + ( + "a fenced sample does not supply a tool link", + TOOLS_HEAD + + "| Tool | Role |\n| --- | --- |\n| [cspell][cspell-link] | Spell checker. |\n" + + TOOLS_DEFS + + "\n```md\n[cspell-link]: https://wrong.example/\n```\n", + 0, + ), + # A tool table shown as a sample inside the section is markup, not entries. + ( + "a fenced tool table is not read as entries", + TOOLS_HEAD + + "| Tool | Role |\n| --- | --- |\n| [cspell][cspell-link] | Spell checker. |\n\n```md\n| [cspell][cspell-link] | wrong description |\n```\n" + + TOOLS_DEFS, + 0, + ), + # The list is alphabetized, which spec/readme-structure.md states and nothing checked until now. + ( + "an unsorted tool list is reported", + TOOLS_HEAD + + "| Tool | Role |\n| --- | --- |\n| [markdownlint-cli2][md-link] | Markdown linter. |\n| [cspell][cspell-link] | Spell checker. |\n" + + TOOLS_DEFS + + "[md-link]: https://github.com/DavidAnson/markdownlint-cli2\n", + 1, + ), + ( + "a sorted tool list is not", + TOOLS_HEAD + + "| Tool | Role |\n| --- | --- |\n| [cspell][cspell-link] | Spell checker. |\n| [markdownlint-cli2][md-link] | Markdown linter. |\n" + + TOOLS_DEFS + + "[md-link]: https://github.com/DavidAnson/markdownlint-cli2\n", + 0, + ), + # GitHub's Markdown makes both outer pipes optional, so a row written without either is an entry rather than a non-row. + # Requiring them read such a table as empty, which scores as clean rather than as unread. + ( + "a row with no trailing pipe is read", + TOOLS_HEAD + + "| Tool | Role |\n| --- | --- |\n| [cspell][cspell-link] | Spell checker.\n" + + TOOLS_DEFS, + 0, + ), + ( + "a row with no trailing pipe is judged like any other", + TOOLS_HEAD + + "| Tool | Role |\n| --- | --- |\n| [cspell][cspell-link] | Spell-checks the README in CI.\n" + + TOOLS_DEFS, + 1, + ), + ( + "a whole table with no trailing pipes is ordered", + TOOLS_HEAD + + "| Tool | Role\n| --- | ---\n| [markdownlint-cli2][md-link] | Markdown linter.\n| [cspell][cspell-link] | Spell checker.\n" + + TOOLS_DEFS + + "[md-link]: https://github.com/DavidAnson/markdownlint-cli2\n", + 1, + ), + ( + "a row with no leading pipe is read", + TOOLS_HEAD + + "Tool | Role |\n--- | --- |\n[cspell][cspell-link] | Spell checker. |\n" + + TOOLS_DEFS, + 0, + ), + ( + "a table with neither outer pipe is judged", + TOOLS_HEAD + + "Tool | Role\n--- | ---\n[cspell][cspell-link] | Spell-checks the README in CI.\n" + + TOOLS_DEFS, + 1, + ), + # The License column is forbidden by spec/readme-structure.md, and project-types.json asserted that as `letter` before anything read it. + # One finding is raised for the table, read off its header, rather than one per row. + ( + "a License column is reported once", + TOOLS_HEAD + + "| Tool | Role | License |\n| --- | --- | --- |\n| [cspell][cspell-link] | Spell checker. | MIT |\n| [markdownlint-cli2][md-link] | Markdown linter. | MIT |\n" + + TOOLS_DEFS + + "[md-link]: https://github.com/DavidAnson/markdownlint-cli2\n", + 1, + ), + ( + "a License column does not stop the rows being read", + TOOLS_HEAD + + "| Tool | Role | License |\n| --- | --- | --- |\n| [cspell][cspell-link] | Spell-checks the README in CI. | MIT |\n" + + TOOLS_DEFS, + 2, + ), + ( + "a table with no License column is not reported", + TOOLS_HEAD + + "| Tool | Role |\n| --- | --- |\n| [cspell][cspell-link] | Spell checker. |\n" + + TOOLS_DEFS, + 0, + ), + # A tool whose own role is the word license must not read as the column, since the header is a header. + ( + "a row cell reading License is not the column", + TOOLS_HEAD + + "| Tool | Role |\n| --- | --- |\n| [Widget][widget-link] | License |\n" + + TOOLS_DEFS, + 0, + ), + ( + "a License column with no outer pipes is reported", + TOOLS_HEAD + + "Tool | Role | License\n--- | --- | ---\n[cspell][cspell-link] | Spell checker. | MIT\n" + + TOOLS_DEFS, + 1, + ), + # A thematic break carries no pipe, so it is not a one-cell delimiter row and the paragraph above it is not a header. + ( + "a thematic break is not a delimiter row", + TOOLS_HEAD + + "Licensing notes follow.\n\n---\n\n| Tool | Role |\n| --- | --- |\n| [cspell][cspell-link] | Spell checker. |\n" + + TOOLS_DEFS, + 0, + ), + # Prose mentioning a tool is not a row, since a link alone does not make one. + ( + "prose naming a cataloged tool is not a row", + TOOLS_HEAD + "We run [cspell][cspell-link] in CI.\n" + TOOLS_DEFS, + 0, + ), +] + +DESC_README = {"README.md": "# Fixture\n\nA short tagline.\n"} + +DESC_CASES = [ + ( + "no declared field, About matches the README tagline", + DESC_README, + {}, + {"description": "A short tagline."}, + 0, + ), + ( + "no declared field, About mismatches the README tagline", + DESC_README, + {}, + {"description": "Something else."}, + 1, + ), + ( + "declared field matches both README and About", + DESC_README, + {"description": "A short tagline."}, + {"description": "A short tagline."}, + 0, + ), + ( + "declared field present, README tagline diverges from it", + DESC_README, + {"description": "The declared description."}, + {"description": "The declared description."}, + 1, + ), + ( + "declared field present, About diverges from it", + DESC_README, + {"description": "A short tagline."}, + {"description": "Something else."}, + 1, + ), + ( + "declared field present, no README at all, About diverges from it", + {}, + {"description": "The declared description."}, + {"description": "Something else."}, + 1, + ), + ( + "declared field present, no README at all, About already matches it", + {}, + {"description": "The declared description."}, + {"description": "The declared description."}, + 0, + ), + ( + "no declared field and no README, nothing to measure against", + {}, + {}, + {"description": "Anything."}, + 0, + ), + ( + "a non-string declared field is reported rather than crashing", + DESC_README, + {"description": 42}, + {"description": "A short tagline."}, + 1, + ), + ( + "an explicit null declared field is reported rather than read as absent", + DESC_README, + {"description": None}, + {"description": "A short tagline."}, + 1, + ), + ( + "a whitespace-only declared field is reported rather than read as absent", + DESC_README, + {"description": " "}, + {"description": "A short tagline."}, + 1, + ), + ( + "a padded declared field is a DEFECT rather than silently trimmed", + DESC_README, + {"description": " A short tagline. "}, + {"description": "A short tagline."}, + 1, + ), + ( + "a declared field with an embedded newline is a DEFECT rather than silently accepted", + DESC_README, + {"description": "A short tagline.\nA second line."}, + {"description": "A short tagline."}, + 1, + ), + ( + "a declared field carrying a Markdown link is a DEFECT rather than silently canonical", + DESC_README, + {"description": "See [docs](https://example.test) for more."}, + {"description": "A short tagline."}, + 1, + ), + ( + "a declared field over the 100-char cap is a DEFECT rather than silently canonical", + DESC_README, + {"description": "a" * 101}, + {"description": "A short tagline."}, + 1, + ), +] + + +def holding(*paths: str) -> Callable[..., Any]: + def fake(path: str, ok404: bool = False) -> Any: + if "/commits?path=" in path: + name = path.split("/commits?path=", 1)[1].split("&", 1)[0] + stamp = {"commit": {"committer": {"date": "2000-01-01T00:00:00+00:00"}}} + return [stamp] if name in paths else [] + name = path.split("/contents/", 1)[1].split("?", 1)[0] + return {"encoding": "base64", "content": "eA==", "name": name} if name in paths else None + + return fake + + +ALT_CASES = [ + (("codecov.yml",), ("codecov.yml", "codecov.yml"), "only the declared path"), + ((".codecov.yml",), (".codecov.yml", ".codecov.yml"), "only the alternative"), + (("codecov.yml", ".codecov.yml"), ("codecov.yml", "codecov.yml"), "both, declared wins"), + ((), ("codecov.yml", None), "neither"), +] +BASELINE_UNIT = {"path": "codecov.yml", "fidelity": "intent", "intentRef": "WORKFLOW.md"} + + +class DriftnoteCase(unittest.TestCase): + """The driftNote freshness rules, driven directly rather than through audit_repo.""" + + def test_note_cases(self) -> None: + for label, types, notes, open_count, wanted in NOTE_CASES: + with self.subTest(label): + got = audit.driftnote_findings( + {"types": types, "driftNotes": notes}, NOTE_SPEC, open_count + ) + texts = [t for _, t in got] + self.assertTrue( + len(texts) == len(wanted) and all(w in t for w, t in zip(wanted, texts)), + f"driftNote {label} -> {texts}, want {wanted}", + ) + + def test_missing_catalog_raises_and_names_the_file(self) -> None: + with self.assertRaises(KeyError, msg="missing catalog: no error raised") as ctx: + audit.driftnote_findings( + {"types": ["hugo"], "driftNotes": ["A note (hugo.build.strict)."]}, + {"registry": {}}, + 0, + ) + self.assertIn( + "project-types.json", + str(ctx.exception), + f"missing catalog: error does not name the file -> {ctx.exception}", + ) + + +class ReadmeCase(unittest.TestCase): + """The README structure engine, run against the real declared model.""" + + def setUp(self) -> None: + self.rm = audit.load("spec/readme-sections.json") + + def test_section_cases(self) -> None: + for label, text, sel_types, public, wantn in README_CASES: + with self.subTest(label): + got = audit.readme_section_findings(text, self.rm, sel_types, public) + self.assertEqual(len(got), wantn, f"readme sections: {label} -> {got}") + + def test_shield_cases(self) -> None: + for case in SHIELD_CASES: + label, text, ent, wantn = case[:4] + want_text = case[4] if len(case) > 4 else None + with self.subTest(label): + got = audit.readme_shield_findings(text, self.rm, ent) + good = len(got) == wantn and ( + want_text is None or any(want_text in t for _, t in got) + ) + self.assertTrue(good, f"readme shields: {label} want={wantn} {want_text} -> {got}") + + def test_link_cases(self) -> None: + for label, text, want_letter, want_drift in LINK_CASES: + with self.subTest(label): + got = audit.readme_link_findings(text, self.rm, "o/r") + gl = sum(1 for k, _ in got if k == "LETTER") + gd = sum(1 for k, _ in got if k == "DRIFT") + self.assertEqual( + (gl, gd), (want_letter, want_drift), f"readme links: {label} {got}" + ) + + def test_tool_cases(self) -> None: + cat = audit.load("spec/third-party-tools.json") + for label, text, wantn in TOOL_CASES: + with self.subTest(label): + got = audit.third_party_tool_findings(text, cat) + self.assertEqual(len(got), wantn, f"3rd party tools: {label} -> {got}") + + def test_canonical_link_entry_with_neither_repopath_nor_match_raises(self) -> None: + broken = dict( + self.rm, canonicalLinks=list(self.rm["canonicalLinks"]) + [{"name": "orphan-link"}] + ) + with self.assertRaises( + KeyError, + msg="canonicalLinks entry with neither repoPath nor match: no error raised", + ) as ctx: + audit.canonical_link_entry("https://example.test/", broken, "o/r") + message = str(ctx.exception) + self.assertTrue( + "readme-sections.json" in message and "orphan-link" in message, + f"canonicalLinks entry error does not name the file and entry -> {ctx.exception}", + ) + + def test_tagline_is_the_first_line_of_the_intro_region(self) -> None: + two_para = audit.title_and_intro( + "# X\n\nThe tagline.\n\nA clarifying paragraph.\n\n## Next\n" + )[1] + self.assertEqual(audit.tagline(two_para), "The tagline.") + self.assertNotIn("\n", audit.tagline(two_para)) + + +class DescriptionCase(unittest.TestCase): + """description_findings(): the README, About, and Docker Hub mirror set.""" + + def test_description_cases(self) -> None: + for label, doc_texts_fx, entry_fx, live_fx, wantn in DESC_CASES: + with self.subTest(label): + got = audit.description_findings(doc_texts_fx, entry_fx, live_fx, "owner/Fixture") + self.assertEqual(len(got), wantn, f"description: {label} -> {got}") + + def test_null_declared_field_is_a_defect_via_validates_contract(self) -> None: + null_declared = audit.description_findings( + DESC_README, + {"description": None}, + {"description": "A short tagline."}, + "owner/Fixture", + ) + self.assertTrue( + any( + k == "DEFECT" and "description must be a non-empty string" in t + for k, t in null_declared + ), + f"description: null-declared-field DEFECT contract -> {null_declared}", + ) + + def test_declared_field_defect_names_the_repo(self) -> None: + null_declared = audit.description_findings( + DESC_README, + {"description": None}, + {"description": "A short tagline."}, + "owner/Fixture", + ) + self.assertTrue( + any(k == "DEFECT" and t.startswith("owner/Fixture:") for k, t in null_declared), + f"description: DEFECT does not name the repo -> {null_declared}", + ) + + def test_declared_field_names_the_registry_as_the_source(self) -> None: + declared_mismatch = audit.description_findings( + DESC_README, + {"description": "The declared description."}, + {"description": "Something else."}, + "owner/Fixture", + ) + self.assertTrue( + any( + "declared description" in t and "registry/repos.json" in t + for _, t in declared_mismatch + ), + f"description: declared-field wording -> {declared_mismatch}", + ) + + def test_stale_docker_hub_short_description_is_reported(self) -> None: + with mock.patch.object( + audit, "docker_hub_description", lambda slug: "A stale Docker Hub blurb." + ): + docker_mismatch = audit.description_findings( + DESC_README, + {"description": "A short tagline.", "publish": [{"target": "docker"}]}, + {"description": "A short tagline."}, + "owner/Fixture", + ) + self.assertTrue( + any("Docker Hub short description" in t for _, t in docker_mismatch), + f"description: Docker Hub mismatch not reported -> {docker_mismatch}", + ) + + +class GroundTruthCase(unittest.TestCase): + """audit_repo, hub_only_paths, gap_dispositions, repo_tree, and baseline file reads.""" + + def test_missing_ground_branch_is_one_error(self) -> None: + entry = {"name": "Fixture", "url": "https://github.com/owner/Fixture", "hasDevelop": True} + with mock.patch.object( + audit, + "gh", + lambda path, ok404=False: None if "/branches/" in path else {"private": False}, + ): + missing_findings, missing_sha = audit.audit_repo( + entry, {"registry": {}}, "no-such-branch" + ) + kinds = [k for k, _ in missing_findings] + errors = [t for k, t in missing_findings if k == "ERROR"] + self.assertTrue( + missing_sha == "" + and len(errors) == 1 + and "no-such-branch" in errors[0] + and kinds == ["DEFECT", "DRIFT", "ERROR"], + f"missing ground branch -> {missing_findings}", + ) + + def test_hub_only_set_excludes_every_declared_path(self) -> None: + declared = {e["path"] for e in audit.load("spec/files.json")["baseline"]} + hub_only = audit.hub_only_paths({"files": audit.load("spec/files.json")}, rev="HEAD") + leaked = sorted(declared & hub_only) + self.assertTrue( + not leaked and hub_only, + f"hub-only set -> {len(hub_only)} paths, {len(leaked)} declared leaked: {leaked}", + ) + + def test_gap_dispositions_drop_malformed_rows(self) -> None: + led = { + "divergences": { + "gaps": [ + {"path": "a/b.sh", "disposition": "retire", "reason": "r"}, + {"path": "c/d.md", "disposition": "accepted"}, + {"path": "no-disposition.md"}, + {"disposition": "retire"}, + "not-an-object", + ] + } + } + got_disp = audit.gap_dispositions(led) + self.assertEqual( + got_disp, + {"a/b.sh": ("retire", "r"), "c/d.md": ("accepted", "")}, + f"gap disposition parsing -> {got_disp}", + ) + + def test_repo_tree_returns_none_unless_whole(self) -> None: + head = {"commit": {"commit": {"tree": {"sha": "deadbeef"}}}} + with mock.patch.object( + audit, + "gh", + lambda path, ok404=False: { + "tree": [{"path": "x", "type": "blob"}], + "truncated": True, + }, + ): + truncated = audit.repo_tree("o/r", head) + with mock.patch.object( + audit, + "gh", + lambda path, ok404=False: { + "tree": [{"path": "x", "type": "blob"}, {"path": "d", "type": "tree"}], + "truncated": False, + }, + ): + whole = audit.repo_tree("o/r", head) + self.assertTrue( + truncated is None and audit.repo_tree("o/r", {"commit": {}}) is None and whole == {"x"}, + f"repo_tree -> truncated={truncated}, whole={whole}", + ) + + def test_read_baseline_file_alternatives(self) -> None: + for held, (want_path, want_name), label in ALT_CASES: + with self.subTest(label): + with mock.patch.object(audit, "gh", holding(*held)): + got_path, got = audit.read_baseline_file( + "o/r", "main", "codecov.yml", [".codecov.yml"] + ) + got_name = got["name"] if got is not None else None + self.assertEqual( + (got_path, got_name), + (want_path, want_name), + f"read_baseline_file, {label} -> {got_path}, {got_name}", + ) + + def test_baseline_file_findings_only_the_alternative(self) -> None: + with ( + mock.patch.object( + audit, "hub_last_change", lambda rel: ("2020-01-01T00:00:00+00:00", "abc1234") + ), + mock.patch.object(audit, "canonical_current_text", lambda rel: "canonical"), + mock.patch.object(audit, "gh", holding(".codecov.yml")), + ): + alt_content, _, alt_findings = audit.baseline_file_findings( + "o/r", "main", "codecov.yml", [".codecov.yml"], BASELINE_UNIT, False + ) + stale_at_alt = [t for lvl, t in alt_findings if lvl == "DRIFT" and ".codecov.yml" in t] + self.assertTrue( + alt_content is not None and len(alt_findings) == 1 and len(stale_at_alt) == 1, + f"baseline_file_findings, only the alternative -> {alt_findings}", + ) + + def test_baseline_file_findings_neither(self) -> None: + with ( + mock.patch.object( + audit, "hub_last_change", lambda rel: ("2020-01-01T00:00:00+00:00", "abc1234") + ), + mock.patch.object(audit, "canonical_current_text", lambda rel: "canonical"), + mock.patch.object(audit, "gh", holding()), + ): + _, _, none_findings = audit.baseline_file_findings( + "o/r", "main", "codecov.yml", [".codecov.yml"], BASELINE_UNIT, False + ) + want_none = [ + ( + "LETTER", + "file: codecov.yml or .codecov.yml absent on main (verify intent per AUDIT.md section 7)", + ) + ] + self.assertEqual( + none_findings, want_none, f"baseline_file_findings, neither -> {none_findings}" + ) + + def test_declared_alternatives_on_the_live_manifest(self) -> None: + live_alts = audit.declared_alternatives(audit.load("spec/files.json")["baseline"]) + self.assertEqual( + live_alts.get("codecov.yml"), + [".codecov.yml"], + f"declared_alternatives on the live manifest -> {live_alts}", + ) + + +class CoverageTypesCase(unittest.TestCase): + """Holds the coverage-candidate and coverage-claiming type helpers.""" + + def test_coverage_candidate_types(self) -> None: + candidate_cases = [ + ( + ["csharp", "python"], + {"csharp": "lint-only", "python": "lint-only"}, + ["python"], + "only Python's lint-only owes coverage", + ), + (["csharp"], {"csharp": "build"}, ["csharp"], "a build language stays a candidate"), + (["csharp"], None, ["csharp"], "a malformed profiles value is read as none"), + ] + for types_in, profiles_in, expected, label in candidate_cases: + with self.subTest(label): + got = audit.coverage_candidate_types(types_in, profiles_in) + self.assertEqual(got, expected) + + def test_coverage_claiming_types(self) -> None: + type_mechs = {"csharp": "codecov", "python": "codecov"} + coverage_cases = [ + (["python"], {"src/pkg/__init__.py", "pyproject.toml"}, [], "python, no tests"), + (["python"], {"tests/test_a.py", "pyproject.toml"}, ["python"], "python with tests"), + ( + ["python"], + {"tests/test_a.py"}, + ["python"], + "python lint-only, with tests, owes the claim same as build", + ), + (["csharp"], {"src/App/App.csproj"}, [], "csharp, no test project"), + (["csharp"], {"test/App.Tests/App.Tests.csproj"}, ["csharp"], "csharp with tests"), + (["csharp"], {"src/Tests/App.csproj"}, ["csharp"], "csharp, located under Tests/"), + ( + ["csharp"], + {"test/Specs.csproj"}, + ["csharp"], + "csharp, located as a test, named nothing", + ), + ( + ["csharp"], + {"src/App/App.csproj", "docs/x.md"}, + [], + "csharp, neither named nor located as a test", + ), + ( + ["python"], + {"test_app.py", "pyproject.toml"}, + ["python"], + "python, root-level test module", + ), + ( + ["python"], + {"app_test.py", "pyproject.toml"}, + ["python"], + "python, trailing _test module", + ), + ( + ["python"], + {"src/app.py", "contest.py"}, + [], + "python, a module merely containing test", + ), + ( + ["python"], + {"tests/README.md", "pyproject.toml"}, + [], + "python, a test directory holding no Python", + ), + ( + ["python"], + {"tests/conftest.py", "pyproject.toml"}, + ["python"], + "python, a test directory holding Python", + ), + ( + ["csharp"], + {"test/App.UnitTest.csproj"}, + ["csharp"], + "csharp, singular Test in the name", + ), + ( + ["csharp"], + {"src/widget.tests.csproj"}, + ["csharp"], + "csharp, a lowercase test segment outside a test directory", + ), + ( + ["csharp"], + {"src/Contest.csproj", "src/Latest.csproj"}, + [], + "csharp, names carrying those letters inside a word rather than as a segment", + ), + ( + ["python"], + {"test/test_a.py", "pyproject.toml"}, + ["python"], + "python, singular test dir", + ), + (["python"], {"src/tests/test_a.py"}, ["python"], "python, tests below the root"), + (["python"], {"tests"}, [], "python, a file named tests is not a directory"), + ( + ["csharp", "python"], + {"test/App.Tests/App.Tests.csproj", "tools/py/pyproject.toml"}, + ["csharp"], + "mixed repo, the tested C# side claims coverage, the untested lint-only python side does not", + ), + (["python"], None, ["python"], "unreadable tree keeps the claim"), + (["docker"], {"tests/test_a.py"}, [], "a type mapped to no mechanism claims nothing"), + ] + for types_in, tree_in, expected, label in coverage_cases: + with self.subTest(label): + got = audit.coverage_claiming_types(types_in, type_mechs, tree_in) + self.assertEqual(got, expected) + + +class PythonTypeFindingsCase(unittest.TestCase): + """Holds the advisory for a tracked Python file in a repo with no python type.""" + + def test_python_type_findings(self) -> None: + # Constructed repo shapes throughout, never a registry entry. + python_type_cases = [ + ( + {"types": ["docs"]}, + {"tools/script.py"}, + 1, + "untyped python: a tracked .py file, no 'python' type", + ), + ( + {"types": ["python"]}, + {"tools/script.py"}, + 0, + "typed python: 'python' already declared", + ), + ({"types": []}, None, 0, "unreadable tree: presence is unanswerable"), + ({"types": []}, {"README.md"}, 0, "no .py file anywhere: nothing to flag"), + ({"types": None}, {"tools/script.py"}, 1, "a malformed types value reads as untyped"), + ( + {"types": ["docs"]}, + {".husky/hub_fetch_run.py"}, + 0, + "the carried hook helper is not the repo's Python", + ), + ( + {"types": ["docs"]}, + {".husky/hub-fetch-run.py"}, + 0, + "the carried hook helper under its former name is not the repo's Python", + ), + ( + { + "types": ["docs"], + "driftNotes": ["Vendored tool, not tracked (python.directories.declared)."], + }, + {"tools/script.py"}, + 0, + "a driftNote naming the check suppresses the advisory", + ), + ( + { + "types": ["docs"], + "driftNotes": ["Retired helper (python.directories.declared)."], + }, + {"README.md"}, + 1, + "a suppression with no .py file left is stale", + ), + ( + { + "types": ["docs"], + "driftNotes": ["python.directories.declared without parentheses"], + }, + {"tools/script.py"}, + 1, + "only the parenthesized id suppresses", + ), + ] + for entry_in, tree_in, expected, label in python_type_cases: + with self.subTest(label): + got = len(audit.python_type_findings(entry_in, tree_in)) + self.assertEqual(got, expected) + + +class FileNamingCase(unittest.TestCase): + """Holds the file naming findings, their driftNote exemptions, and the capped listing.""" + + def test_file_naming_cases(self) -> None: + plain = "100644" + exe = "100755" + file_naming_cases: list[tuple[Any, ...]] = [ + ( + "every conforming shape passes", + { + "scripts/fetch_report.py": plain, + "pkg/__init__.py": plain, + "tests/conftest.py": plain, + "scripts/sync-tools.sh": exe, + "scripts/sync-tools.ps1": plain, + "scripts/sync-tools.cmd": plain, + "scripts/sync-tools.bat": plain, + "tests/sync-tools.Tests.ps1": plain, + ".husky/pre-commit": exe, + "bin/run-backup": exe, + "PSScriptAnalyzerSettings.psd1": plain, + "Makefile": plain, + "README.md": plain, + ".envrc": exe, + }, + {}, + set(), + set(), + ), + ( + "a hyphenated or capitalized .py breaks", + { + "scripts/fetch-report.py": plain, + "scripts/FetchReport.py": plain, + "tool.PY": plain, + }, + {}, + {"scripts/fetch-report.py", "scripts/FetchReport.py", "tool.PY"}, + set(), + ), + ( + "the carried hook helper under its former name breaks, beside the current name or alone", + {".husky/hub-fetch-run.py": plain, ".husky/hub_fetch_run.py": plain}, + {}, + {".husky/hub-fetch-run.py"}, + set(), + ), + ( + "a snake_case or capitalized shell, PowerShell, or batch name with no wrapped tool breaks", + { + "scripts/sync_tools.sh": exe, + "scripts/Sync-Tools.ps1": plain, + "scripts/SYNC.CMD": plain, + "scripts/sync-tools.SH": exe, + "scripts/sync_tools.bat": plain, + }, + {}, + { + "scripts/sync_tools.sh", + "scripts/Sync-Tools.ps1", + "scripts/SYNC.CMD", + "scripts/sync-tools.SH", + "scripts/sync_tools.bat", + }, + set(), + ), + ( + "a Pester suffix over a non-conforming stem breaks", + {"tests/Sync_Tools.Tests.ps1": plain}, + {}, + {"tests/Sync_Tools.Tests.ps1"}, + set(), + ), + ( + "a launcher beside its Python tool keeps the tool's stem", + { + "scripts/fetch_report.py": plain, + "scripts/fetch_report.sh": exe, + "scripts/fetch_report.ps1": plain, + "scripts/fetch_report.cmd": plain, + }, + {}, + set(), + {}, + ), + ( + "a launcher whose Python tool sits in another directory is undecided", + {"tools/fetch_report.py": plain, "scripts/fetch_report.sh": exe}, + {}, + set(), + {"scripts/fetch_report.sh"}, + ), + ( + "an extensionless launcher beside its Python tool may be an installed command, with no read", + {"scripts/fetch_report.py": plain, "scripts/fetch_report": exe}, + {}, + set(), + {"scripts/fetch_report"}, + ), + ( + "an extensionless launcher whose Python tool sits elsewhere is undecided, with no read", + {"tools/fetch_report.py": plain, "bin/fetch_report": exe}, + {}, + set(), + {"bin/fetch_report"}, + ), + ( + "an extensionless executable with a shebang and a non-conforming name breaks", + {"bin/run_backup": exe, "bin/RunBackup": exe}, + {"bin/run_backup": "#!/usr/bin/env bash\n", "bin/RunBackup": "#!/bin/sh\n"}, + {"bin/run_backup", "bin/RunBackup"}, + set(), + ), + ( + "an extensionless executable with no shebang, or no readable text, is undecided", + {"bin/Tool_Binary": exe, "bin/Unreadable_Tool": exe}, + {"bin/Tool_Binary": "\x7fELF", "bin/Unreadable_Tool": None}, + set(), + {"bin/Tool_Binary", "bin/Unreadable_Tool"}, + ), + ( + "an extensionless file without the executable bit is out of scope", + {"docs/Release_Notes": plain}, + {}, + set(), + set(), + ), + ( + "an executable dotfile and a dotted name are not judged, with no read", + {"bin/.Hidden_Hook": exe, "bin/Run_Me.v2": exe}, + {}, + set(), + set(), + ), + ( + "a dotfile with a listed extension is judged on its stem", + {"bin/.Hook_Run.sh": exe, "bin/.lint.py": plain}, + {}, + {"bin/.Hook_Run.sh", "bin/.lint.py"}, + set(), + ), + ] + for label, blobs_in, texts_in, want_broken, want_undecided in file_naming_cases: + with self.subTest(label): + reads: list[str] = [] + + def read_text(path, texts_in=texts_in, reads=reads): + reads.append(path) + return texts_in.get(path) + + got = audit.file_naming_findings({}, blobs_in, read_text) + broken_text = next((t for _, t in got if " break " in t), "") + undecided_text = next((t for _, t in got if "cannot decide" in t), "") + good = ( + all(kind == "DRIFT" for kind, _ in got) + and len(got) == bool(want_broken) + bool(want_undecided) + and broken_text.startswith( + f"file-naming: {len(want_broken)} file(s)" if want_broken else "" + ) + and all( + f"{p}," in f"{broken_text}," or f"{p} " in broken_text for p in want_broken + ) + and undecided_text.startswith( + f"file-naming: {len(want_undecided)} file(s)" if want_undecided else "" + ) + and all(f"{p} (" in undecided_text for p in want_undecided) + and set(reads) == set(texts_in) + ) + self.assertTrue(good, f"{got}, reads {reads}") + + def test_file_naming_capped(self) -> None: + plain = "100644" + many = {f"scripts/Tool{n}.py": plain for n in range(7)} + capped = audit.file_naming_findings({}, many, lambda path: None) + self.assertTrue( + len(capped) == 1 and capped[0][1].endswith(" and 2 more"), + f"past five paths the list is capped -> {capped}", + ) + + def test_file_naming_exemptions(self) -> None: + plain = "100644" + exe = "100755" + exempt_note = ( + "A package format fixes `tools/Install.ps1` by that name (filenaming.scripts)." + ) + exemption_blobs = { + "tools/Install.ps1": plain, + "tools/Setup_Host.sh": exe, + "bin/Tool_Binary": exe, + } + exemption_cases = [ + ( + "a named path is exempt and an unnamed break still reports", + [exempt_note], + exemption_blobs, + {"tools/Setup_Host.sh"}, + {"bin/Tool_Binary"}, + [], + ), + ( + "an undecided path can be exempted too", + [ + "An installed command fixed by its vendor, `bin/Tool_Binary` (filenaming.scripts)." + ], + exemption_blobs, + {"tools/Install.ps1", "tools/Setup_Host.sh"}, + set(), + [], + ), + ( + "a note naming a path that no longer breaks, or no longer exists, is stale", + [ + "Tool-fixed `tools/Install.ps1`, `tools/install-host.sh`, `tools/Gone.ps1` (filenaming.scripts)." + ], + {"tools/Install.ps1": plain, "tools/install-host.sh": exe}, + set(), + set(), + ["exempts 2 path(s)", "tools/install-host.sh, tools/Gone.ps1"], + ), + ( + "a note naming no path exempts nothing and is reported", + ["Install scripts are named by the package format (filenaming.scripts)."], + {"tools/Install.ps1": plain}, + {"tools/Install.ps1"}, + set(), + ["names no path"], + ), + ( + "a path without the parenthesized id exempts nothing", + ["A package format fixes `tools/Install.ps1` (filenaming.scripts is pending)."], + {"tools/Install.ps1": plain}, + {"tools/Install.ps1"}, + set(), + [], + ), + ] + for label, notes_in, blobs_in, want_broken, want_undecided, want_notes in exemption_cases: + with self.subTest(label): + got = audit.file_naming_findings( + {"driftNotes": notes_in}, blobs_in, lambda path: "\x7fELF" + ) + note_texts = [t for _, t in got if t.startswith("registry: ")] + broken_text = next((t for _, t in got if " break " in t), "") + undecided_text = next((t for _, t in got if "cannot decide" in t), "") + good = ( + all(kind == "DRIFT" for kind, _ in got) + and len(got) == len(note_texts) + bool(want_broken) + bool(want_undecided) + and len(note_texts) == (1 if want_notes else 0) + and all(w in note_texts[0] for w in want_notes) + and all(p in broken_text for p in want_broken) + and all(p not in broken_text for p in blobs_in.keys() - want_broken) + and all(p in undecided_text for p in want_undecided) + and all(p not in undecided_text for p in blobs_in.keys() - want_undecided) + ) + self.assertTrue(good, f"{got}") + + def test_exemption_note_is_not_a_check_to_judge(self) -> None: + exempt_note = ( + "A package format fixes `tools/Install.ps1` by that name (filenaming.scripts)." + ) + self.assertEqual( + audit.driftnote_findings( + {"types": ["docs"], "driftNotes": [exempt_note]}, NOTE_SPEC, 0 + ), + [], + "an exemption note is not surfaced as a check to judge", + ) + + def test_unreadable_tree_reports_nothing(self) -> None: + self.assertEqual( + audit.file_naming_findings({}, None, lambda path: None), + [], + "an unreadable tree reports nothing here", + ) + + +class SecretStemCase(unittest.TestCase): + """Holds the runtime secret file stem findings.""" + + def test_secret_stem_cases(self) -> None: + secret_stem_cases = [ + ( + "conforming stems, examples, and the directory's own files pass", + { + ".secrets/README.md", + ".secrets/.gitignore", + ".secrets/db_root_password.example", + ".secrets/mail_relay.env.example", + "deploy/.secrets/service_account.json.example", + "docs/Mixed-Case.md", + }, + set(), + ), + ( + "a hyphenated, capitalized, or bare stem breaks", + { + ".secrets/db-root-password.example", + ".secrets/Mail_Relay.env.example", + ".secrets/.env.example", + "deploy/.secrets/api-token", + }, + { + ".secrets/db-root-password.example", + ".secrets/Mail_Relay.env.example", + ".secrets/.env.example", + "deploy/.secrets/api-token", + }, + ), + ( + "a file outside a .secrets directory, or nested below one, is out of scope", + { + "secrets/db-root-password.example", + "config/.secrets-notes/Odd-Name", + ".secrets/docs/Mixed-Case.md", + }, + set(), + ), + ] + for label, tree_in, want in secret_stem_cases: + with self.subTest(label): + got = audit.secret_stem_findings(tree_in) + good = (not want and got == []) or ( + len(got) == 1 + and got[0][0] == "DRIFT" + and got[0][1].startswith(f"runtime-secrets: {len(want)} secret file(s)") + and all(p in got[0][1] for p in want) + ) + self.assertTrue(good, f"{got}") + + def test_unreadable_tree_reports_nothing(self) -> None: + self.assertEqual( + audit.secret_stem_findings(None), [], "an unreadable tree reports nothing here" + ) + + +class DeclaredTypesCase(unittest.TestCase): + """Holds the reading of a registry entry's types value.""" + + def test_declared_types(self) -> None: + declared_types_cases = [ + ({"types": ["python", "docs"]}, ["python", "docs"], "a list of strings is read whole"), + ({"types": ["python", 3, None]}, ["python"], "a mixed list keeps its string elements"), + ({"types": "python"}, [], "a string is not a list"), + ({"types": {"python": 1}}, [], "an object is not a list"), + ({"types": None}, [], "a declared null reads as untyped"), + ({}, [], "an absent field reads as untyped"), + ] + for entry_in, expected, label in declared_types_cases: + with self.subTest(label): + got = audit.declared_types(entry_in) + self.assertEqual(got, expected) + + +class PythonDirectoriesCase(unittest.TestCase): + """Holds the python directory coverage, undeclared root, and caller input findings.""" + + def test_python_directory_coverage(self) -> None: + python_directory_cases: list[tuple[Any, ...]] = [ + ( + {"pythonDirectories": ["Tools"]}, + {"Tools/a.py", "Extra/b.py"}, + 1, + "declared directory: one file outside it is missed", + ), + ( + {"pythonDirectories": ["Tools"]}, + {"Tools/a.py", "Tools/sub/b.py"}, + 0, + "declared directory: a nested file is still covered", + ), + ( + {}, + {"tools/a.py"}, + 1, + "no declaration and no root pyproject.toml: nothing is expected to cover it", + ), + ( + {}, + {"pyproject.toml", "a.py", "sub/b.py"}, + 0, + "no declaration, root pyproject.toml: '.' fallback covers everything", + ), + ( + {"pythonDirectories": ["Tools"]}, + None, + 0, + "unreadable tree: coverage is unanswerable", + ), + ( + {"pythonDirectories": ["Tools"]}, + {f"Extra/f{i}.py" for i in range(7)}, + 1, + "more than 5 missed: one finding, capped listing", + ), + ] + # Each case describes a Python-typed repo unless it names its own types, since an untyped one gets only the type advisory. + python_directory_cases.append( + ({"types": ["docs"]}, {"tools/x.py"}, 0, "an untyped repo gets only the type advisory") + ) + python_directory_cases.append( + ( + {"pythonDirectories": ["Tools"]}, + {"Tools/a.py", "hub_fetch_run.py"}, + 0, + "the carried hook helper sits outside every directory", + ) + ) + python_directory_cases += [ + ({"types": "python"}, {"tools/x.py"}, 0, "a string types value reads as untyped"), + ({"types": None}, {"tools/x.py"}, 0, "a null types value reads as untyped"), + ] + for entry_in, tree_in, expected, label in python_directory_cases: + with self.subTest(label): + got_findings = audit.python_directory_coverage_findings( + {"types": ["python"], **entry_in}, tree_in + ) + self.assertEqual(len(got_findings), expected) + if expected and len(tree_in or []) > 5: + self.assertIn( + "more", + got_findings[0][1], + "missing the '... and N more' cap", + ) + + def test_python_undeclared_root(self) -> None: + undeclared_root_cases: list[tuple[Any, ...]] = [ + ( + {}, + {"pyproject.toml", "tests/test_a.py", "a.py"}, + 1, + "a lint-only root suite nothing runs", + ), + ( + {}, + {"pyproject.toml", "tests/test_a.py", "uv.lock"}, + 0, + "a locked root runs by default", + ), + ( + {}, + {"pyproject.toml", "tests/test_a.py", "requirements.txt"}, + 0, + "a pip root runs by default", + ), + ( + {}, + {"pyproject.toml", "tests/test_a.py", "docs/requirements.txt"}, + 1, + "a nested manifest is not the root's", + ), + ( + {"pythonDirectories": ["."]}, + {"pyproject.toml", "tests/test_a.py"}, + 0, + "a declared root runs", + ), + ({}, {"pyproject.toml", "a.py"}, 0, "no root suite to run"), + ( + {}, + {"pyproject.toml", "tests/App.Tests/App.Tests.csproj"}, + 0, + "a tests/ holding no Python", + ), + ({}, None, 0, "unreadable tree reports nothing"), + ] + undeclared_root_cases.append( + ( + {"types": ["docs"]}, + {"pyproject.toml", "tests/test_a.py"}, + 0, + "an untyped repo gets only the type advisory", + ) + ) + undeclared_root_cases += [ + ( + {"types": "python"}, + {"pyproject.toml", "tests/test_a.py"}, + 0, + "a string types value reads as untyped", + ), + ( + {"types": None}, + {"pyproject.toml", "tests/test_a.py"}, + 0, + "a null types value reads as untyped", + ), + ] + for entry_in, tree_in, expected, label in undeclared_root_cases: + with self.subTest(label): + got = len( + audit.python_undeclared_root_findings( + {"types": ["python"], **entry_in}, tree_in + ) + ) + self.assertEqual(got, expected) + + def test_python_directories_caller(self) -> None: + py_caller_uses = " uses: ./.github/workflows/validate-task.yml\n" + py_caller_block = ( + "jobs:\n validate:\n" + + py_caller_uses + + " with:\n python-directories: |\n Tools\n" + ) + py_caller_plain = ( + "jobs:\n validate:\n" + py_caller_uses + " with:\n python-directories: Tools\n" + ) + py_caller_no_with = "jobs:\n validate:\n" + py_caller_uses + py_caller_no_hub = "jobs:\n validate:\n runs-on: ubuntu-latest\n steps:\n - run: echo inline lint\n" + py_caller_path = ".github/workflows/test-pull-request.yml" + python_caller_cases = [ + ( + py_caller_path, + py_caller_block, + {"pythonDirectories": ["Tools"]}, + 0, + "block scalar matches the registry", + ), + ( + py_caller_path, + py_caller_block, + {"pythonDirectories": ["Other"]}, + 1, + "block scalar disagrees with the registry", + ), + ( + py_caller_path, + py_caller_plain, + {"pythonDirectories": ["Tools"]}, + 0, + "plain single value matches the registry", + ), + ( + py_caller_path, + py_caller_no_with, + {}, + 0, + "no with: input, registry declares none - both empty", + ), + ( + py_caller_path, + py_caller_no_with, + {"pythonDirectories": ["Tools"]}, + 1, + "no with: input, registry declares one", + ), + ( + ".github/workflows/other.yml", + py_caller_block, + {"pythonDirectories": ["Other"]}, + 0, + "not a scoped caller path", + ), + ( + py_caller_path, + py_caller_no_hub, + {"pythonDirectories": ["Tools"]}, + 0, + "validate job never reaches validate-task.yml", + ), + ( + py_caller_path, + py_caller_block.replace(" validate:\n", " check:\n"), + {"pythonDirectories": ["Tools"]}, + 1, + "a renamed job is reported, not skipped", + ), + ( + py_caller_path, + "jobs:\n validate:\n" + + py_caller_uses + + " with: { python-directories: Tools }\n", + {}, + 1, + "a flow-mapping with: is reported, not misread as declaring nothing", + ), + ( + py_caller_path, + py_caller_block + " validate-extra:\n" + py_caller_uses, + {"pythonDirectories": ["Tools"]}, + 1, + "a second job calling validate-task.yml is reported beside a matching validate", + ), + ] + py_caller_root = py_caller_plain.replace( + "python-directories: Tools", "python-directories: ." + ) + python_caller_cases.append( + ( + py_caller_path, + py_caller_root, + {}, + 1, + "naming the root is not the same as naming nothing", + ) + ) + py_caller_folded = py_caller_block.replace( + "python-directories: |\n Tools\n", + "python-directories: >\n Tools\n Other\n", + ) + py_caller_commented = py_caller_plain.replace( + " with:\n", " with:\n # a note indented past the keys\n" + ) + python_caller_cases.append( + ( + py_caller_path, + py_caller_commented, + {"pythonDirectories": ["Tools"]}, + 0, + "a deeper comment hides no key", + ) + ) + py_caller_tabbed = py_caller_plain.replace( + "python-directories: Tools", "python-directories: Too\tls" + ) + py_caller_value = py_caller_plain.replace( + "python-directories: Tools", "python-directories: {}" + ) + python_caller_cases += [ + ( + py_caller_path, + py_caller_value.format('&dirs "Tools\\nOther"'), + {}, + 1, + "an anchored escaped double-quoted value is refused", + ), + ( + py_caller_path, + py_caller_value.format('!!str "Tools\\nOther"'), + {}, + 1, + "a tagged escaped double-quoted value is refused", + ), + ( + py_caller_path, + py_caller_value.format('\n "Tools\\nOther"'), + {}, + 1, + "an escaped double-quoted value on the next line is refused", + ), + ( + py_caller_path, + py_caller_value.format( + '\n # a note at the key column\n "Tools\\nOther"' + ), + {}, + 1, + "a comment at the key column hides no next-line value", + ), + ( + py_caller_path, + py_caller_value.format('"Tools\n \\nOther"'), + {"pythonDirectories": ["Tools"]}, + 1, + "an escape on a double-quoted value's continuation line is refused", + ), + ( + py_caller_path, + py_caller_value.format('"Tools" # a\\b'), + {"pythonDirectories": ["Tools"]}, + 0, + "a backslash past the closing quote is not an escape", + ), + ( + py_caller_path, + py_caller_value.format("&dirs >\n Tools"), + {"pythonDirectories": ["Tools"]}, + 1, + "an anchored folded scalar is refused", + ), + ( + py_caller_path, + py_caller_folded, + {"pythonDirectories": ["Other", "Tools"]}, + 1, + "a folded scalar is refused", + ), + ( + py_caller_path, + py_caller_tabbed, + {"pythonDirectories": ["Too\tls"]}, + 1, + "a tabbed entry resolves to nothing", + ), + ] + for path_in, text_in, entry_in, expected, label in python_caller_cases: + with self.subTest(label): + got = len(audit.python_directories_caller_findings(path_in, text_in, entry_in)) + self.assertEqual(got, expected) + py_caller_escaped = py_caller_plain.replace( + "python-directories: Tools", 'python-directories: "Tools\\nOther"' + ) + escaped_got = audit.python_directories_caller_findings( + py_caller_path, py_caller_escaped, {"pythonDirectories": ["Other", "Tools"]} + ) + self.assertTrue( + len(escaped_got) == 1 and "backslash escape" in escaped_got[0][1], + f"an escaped double-quoted value is refused -> {escaped_got}", + ) + + +class ConcurrencyCallerFindingsCase(unittest.TestCase): + """Holds the publisher and merge-bot concurrency group checks.""" + + def setUp(self) -> None: + + snippet_dir = audit.ROOT / "catalog" / "snippets" / "workflows" + pub_path = ".github/workflows/publish-release.yml" + bot_path = ".github/workflows/merge-bot-pull-request.yml" + pub_text = (snippet_dir / "publish-release.yml").read_text(encoding="utf-8") + bot_text = (snippet_dir / "merge-bot-pull-request.yml").read_text(encoding="utf-8") + pub_group = "group: ${{ github.workflow }}-${{ github.ref }}" + pub_line = next(ln for ln in pub_text.splitlines() if ln.strip().startswith("group:")) + bot_line = next(ln for ln in bot_text.splitlines() if ln.strip().startswith("group:")) + pub_value = pub_line.split("group:", 1)[1].strip() + sched_line = ( + " " + + pub_group + + "${{ github.event_name == 'schedule' && format('-{0}', github.run_id) || '' }}" + ) + pub_gate = ( + "if: ${{ needs.plan.outputs.publish == 'true' && needs.validate.result == 'success' }}" + ) + sched_gate = "if: ${{ github.event_name == 'workflow_dispatch' && needs.plan.outputs.publish == 'true' }}" + sched_text = re.sub( + r"(?s)\non:\n.*?\n(?=\S)", + "\non:\n workflow_dispatch:\n schedule:\n - cron: '0 2 * * MON'\n\n", + pub_text.replace(pub_line, sched_line).replace(pub_gate, sched_gate), + count=1, + ) + self.fx = ( + pub_path, + bot_path, + pub_text, + bot_text, + pub_group, + pub_line, + bot_line, + pub_value, + sched_line, + pub_gate, + sched_gate, + sched_text, + ) + + def test_refusal_messages(self) -> None: + ( + pub_path, + bot_path, + pub_text, + bot_text, + pub_group, + pub_line, + bot_line, + pub_value, + sched_line, + *_, + ) = self.fx + refusal_cases = [ + ( + pub_path, + pub_text.replace(pub_line, sched_line), + "cannot read its schedule", + "a schedule-only suffix names what keeps it from qualifying", + ), + ( + bot_path, + bot_text.replace(bot_line, bot_line + sched_line.split("github.ref }}", 1)[1]), + "group is", + "a merge-bot schedule suffix gets no publisher advice", + ), + ( + pub_path, + "concurrency: ${{ github.workflow }}-${{ github.ref }}\n -extra\njobs:\n x: {}\n", + "cannot read", + "a continuation after an inline scalar", + ), + ( + pub_path, + pub_text.replace(pub_line, " group: >-\n " + pub_value), + "cannot read", + "a folded scalar", + ), + ( + pub_path, + pub_text.replace(pub_line, pub_line + "\n more"), + "cannot read", + "a continuation line", + ), + (pub_path, "concurrency: { group: x }\n", "cannot read", "a flow mapping"), + ( + pub_path, + pub_text.replace("concurrency:\n", "concurrency: &c\n", 1), + "", + "an anchor on the key", + ), + ( + pub_path, + pub_text.replace( + pub_line, + " " + + pub_group + + "${{ github.event_name == 'push' && format('-{0}', github.run_id) || '' }}", + ), + "group is", + "a suffix without the main clause", + ), + ( + pub_path, + pub_text.replace( + pub_line, + " " + pub_group + "${{ format('-{0}{1}', github.run_id, github.event_name) }}", + ), + "group is", + "an unconditional suffix naming event_name", + ), + ] + for path_in, text_in, needle, label in refusal_cases: + with self.subTest(label): + got = audit.concurrency_caller_findings(path_in, text_in) + if needle == "": + self.assertFalse(got, f"{label} -> {got}") + else: + self.assertTrue(any(needle in f[1] for f in got), f"{label} -> {got}") + + def test_finding_counts(self) -> None: + ( + pub_path, + bot_path, + pub_text, + bot_text, + pub_group, + pub_line, + bot_line, + pub_value, + sched_line, + pub_gate, + sched_gate, + sched_text, + ) = self.fx + concurrency_cases = [ + ( + pub_path, + sched_text, + 0, + "a schedule-only suffix on a publisher whose schedule never publishes", + ), + ( + pub_path, + pub_text.replace(pub_line, sched_line), + 1, + "a schedule-only suffix on a push-triggered publisher", + ), + ( + pub_path, + sched_text.replace(sched_gate, pub_gate), + 1, + "a schedule-only suffix on a publisher whose schedule publishes", + ), + ( + pub_path, + sched_text.replace( + sched_gate, + "if: ${{ github.event_name == 'workflow_dispatch' || needs.plan.outputs.publish == 'true' }}", + ), + 1, + "a dispatch gate behind a disjunction", + ), + ( + pub_path, + sched_text.replace( + sched_gate, + "if: >-\n github.event_name == 'workflow_dispatch' &&\n needs.plan.outputs.publish == 'true'", + ), + 0, + "a folded dispatch gate", + ), + ( + pub_path, + sched_text.replace( + sched_gate, + "if: >- # note\n github.event_name == 'workflow_dispatch' &&\n needs.plan.outputs.publish == 'true'", + ), + 0, + "a folded dispatch gate with a comment on its key line", + ), + ( + pub_path, + sched_text.replace("\non:\n", "\non: # triggers\n", 1), + 0, + "a comment on the on: key line", + ), + ( + pub_path, + sched_text.replace( + sched_gate, "if: ${{ github.event_name == 'workflow_dispatch' }}" + ), + 0, + "a dispatch test alone", + ), + ( + pub_path, + sched_text.replace( + sched_gate, + "if: ${{ needs.plan.outputs.publish == 'true' && (github.event_name != 'schedule') }}", + ), + 1, + "a parenthesized not-schedule test as the last conjunct", + ), + ( + pub_path, + sched_text.replace( + sched_gate, + "if: ${{ needs.plan.outputs.publish == 'true' && (github.event_name == 'workflow_dispatch') }}", + ), + 0, + "a parenthesized dispatch test as the last conjunct", + ), + ( + pub_path, + sched_text.replace( + sched_gate, + "if: ${{ !(needs.plan.outputs.publish == 'true' && (github.event_name == 'workflow_dispatch')) }}", + ), + 1, + "a dispatch test inside a negated group", + ), + ( + pub_path, + sched_text.replace("build-release-task.yml@", "other-task.yml@"), + 1, + "a schedule-only suffix with no release job", + ), + ( + pub_path, + sched_text.replace(" schedule:\n", " push:\n schedule:\n", 1), + 1, + "a schedule-only suffix beside a push trigger", + ), + (pub_path, pub_text, 0, "the reference publisher, run_id suffix included"), + (bot_path, bot_text, 0, "the reference merge-bot"), + ( + pub_path, + pub_text.replace( + "github.actor != 'ptr727-codegen[bot]' && github.actor != 'dependabot[bot]'", + "github.actor != 'dependabot[bot]' && github.actor != 'ptr727-codegen[bot]'", + ), + 0, + "the allowlisted actors in the other order", + ), + ( + pub_path, + pub_text.replace(pub_line, " " + pub_group), + 0, + "a bare ref-scoped publisher group", + ), + ( + pub_path, + pub_text.replace(pub_line, " group: ${{ github.workflow }}"), + 1, + "a global publisher group", + ), + ( + pub_path, + pub_text.replace(pub_line, " group: publish"), + 1, + "a literal publisher group", + ), + ( + pub_path, + pub_text.replace(pub_line, " " + pub_group + "-extra"), + 1, + "a literal publisher suffix", + ), + ( + pub_path, + pub_text.replace(pub_line, " " + pub_group + "${{ github.sha }}"), + 1, + "a non-run_id publisher suffix", + ), + ( + pub_path, + pub_text.replace("cancel-in-progress: false", "cancel-in-progress: true"), + 1, + "a cancelling publisher", + ), + ( + bot_path, + bot_text.replace(bot_line, " group: ${{ github.workflow }}-${{ github.ref }}"), + 1, + "a ref-keyed merge-bot", + ), + ( + bot_path, + bot_text.replace("cancel-in-progress: false", "cancel-in-progress: true"), + 1, + "a cancelling merge-bot", + ), + ( + bot_path, + bot_text.replace(bot_line, bot_line + "${{ github.run_id }}"), + 1, + "a run_id suffix is the publisher's alone", + ), + ( + pub_path, + re.sub(r"(?s)\nconcurrency:.*?cancel-in-progress: \w+\n", "\n", pub_text), + 1, + "no concurrency block", + ), + ( + pub_path, + pub_text.replace( + "github.actor != 'dependabot[bot]'", "github.actor == 'dependabot[bot]'" + ), + 1, + "an actor predicate that is not an exclusion list", + ), + ( + pub_path, + pub_text.replace("refs/heads/main", "refs/heads/develop"), + 1, + "a suffix keyed on another branch", + ), + ( + pub_path, + pub_text.replace( + "github.actor != 'dependabot[bot]'", "github.actor != 'other[bot]'" + ), + 1, + "an actor list that is not the plan task's allowlist", + ), + ( + pub_path, + pub_text.replace("ptr727-codegen[bot]", "ptr727-codegen [bot]"), + 1, + "whitespace inside an allowlisted actor name", + ), + ( + bot_path, + bot_text.replace(bot_line, bot_line + pub_value.split("github.ref }}", 1)[1]), + 1, + "the push-only suffix is the publisher's alone", + ), + ( + bot_path, + bot_text.replace(bot_line, bot_line + sched_line.split("github.ref }}", 1)[1]), + 1, + "the schedule-only suffix is the publisher's alone", + ), + ( + pub_path, + pub_text.replace(pub_line, " group: '" + pub_value.replace("'", "''") + "'"), + 0, + "a single-quoted publisher group", + ), + ( + pub_path, + pub_text.replace(pub_line, " group: " + pub_value + " # note"), + 0, + "a trailing comment", + ), + ( + pub_path, + pub_text.replace("concurrency:\n", "concurrency: # note\n", 1), + 0, + "a comment on the key line", + ), + ( + pub_path, + pub_text.replace(pub_line, " group: ${{github.workflow}}-${{github.ref}}"), + 0, + "tight expression spacing", + ), + ( + pub_path, + pub_text.replace("cancel-in-progress: false", "cancel-in-progress: ${{ false }}"), + 0, + "an expression false", + ), + ( + pub_path, + pub_text.replace(pub_line, " group: >-\n " + pub_value), + 1, + "a folded scalar is refused", + ), + ( + pub_path, + pub_text.replace(pub_line, " " + pub_group + "${{ github.run_id }}"), + 1, + "an unconditional run_id suffix", + ), + ( + pub_path, + pub_text.replace( + pub_line, + " " + pub_group + "${{ github.ref == 'refs/heads/main' && github.run_id }}", + ), + 1, + "a suffix naming no event_name", + ), + ( + pub_path, + pub_text.replace( + pub_line, + " " + + pub_group + + "${{ github.sha }}-${{ github.event_name }}${{ github.run_id }}", + ), + 1, + "several suffix expressions", + ), + ( + bot_path, + "concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }}\n", + 0, + "a bare scalar merge-bot group", + ), + ( + pub_path, + "concurrency: ${{ github.workflow }}-${{ github.ref }}\njobs:\n x:\n concurrency:\n group: y\n", + 0, + "a bare scalar group, job-level concurrency ignored", + ), + ( + ".github/workflows/test-pull-request.yml", + "concurrency:\n group: x\n", + 0, + "another workflow is not read", + ), + ] + for path_in, text_in, expected, label in concurrency_cases: + with self.subTest(label): + got = len(audit.concurrency_caller_findings(path_in, text_in)) + self.assertEqual(got, expected) + + +class RepoIdentityCase(unittest.TestCase): + """Holds the repo_identity URL parsing check.""" + + def test_repo_identity(self) -> None: + id_cases = [ + ("https://github.com/owner/Repo", "owner/repo"), + ("https://github.com/owner/Repo/", "owner/repo"), + # Padding still resolves here, since a padded entry must read as an entry that exists. + (" https://github.com/owner/Repo ", "owner/repo"), + ("https://github.com/owner/Repo.git", "owner/repo"), + ("https://github.com/owner/Repo.github", "owner/repo.github"), + ("https://github.com/owner/Repo?tab=readme", None), + ("https://github.com/owner/Repo#readme", None), + ("https://github.com/owner/Repo.git?tab=readme", None), + ("https://github.com/owner?tab=readme/Repo", None), + ("https://github.com/owner#readme/Repo", None), + ("http://github.com/owner/Repo", None), + ("https://gitlab.com/owner/Repo", None), + ("not a url", None), + (None, None), + (42, None), + ] + for url, want in id_cases: + with self.subTest(str(url)): + self.assertEqual(audit.repo_identity(url), want) + + +class OwnerReposCase(unittest.TestCase): + """Holds the owner_repos fork, pagination, and gh failure checks.""" + + def test_forks_dropped_and_pagination_stops_on_short_page(self) -> None: + pages = { + 1: [{"name": f"r{i}", "fork": False} for i in range(100)], + 2: [{"name": "kept", "fork": False}, {"name": "dropped-fork", "fork": True}], + } + fake = lambda path, ok404=False: ( + {"login": "owner"} if path == "user" else pages[int(path.rsplit("page=", 1)[1])] + ) + with mock.patch.object(audit, "gh", fake): + got = {r["name"] for r in audit.owner_repos("owner")} + want = {f"r{i}" for i in range(100)} | {"kept"} + self.assertEqual(got, want, "owner_repos forks/pagination wrong") + + def test_none_user_response_raises_mismatch(self) -> None: + def fake(path: str, ok404: bool = False) -> Any: + return None if path == "user" else [] + + with mock.patch.object(audit, "gh", fake), self.assertRaises(RuntimeError) as ctx: + audit.owner_repos("owner") + self.assertIn("not registry owner 'owner'", str(ctx.exception)) + + def test_differently_cased_login_matches(self) -> None: + fake = lambda path, ok404=False: {"login": "Owner"} if path == "user" else [] + with mock.patch.object(audit, "gh", fake): + audit.owner_repos("owner") + + def test_none_page_fails_loud(self) -> None: + fake = lambda path, ok404=False: {"login": "owner"} if path == "user" else None + with mock.patch.object(audit, "gh", fake), self.assertRaises(RuntimeError): + audit.owner_repos("owner") + + +class MembershipFindingsCase(unittest.TestCase): + """Holds the registry membership findings against a faked owner_repos.""" + + def test_membership_findings(self) -> None: + cases = [ + ( + "a repo missing from the registry is a DEFECT", + [{"name": "New", "full_name": "owner/New", "archived": False}], + {"owner": "owner", "repos": []}, + [("DEFECT", "owner/New: exists on GitHub with no registry/repos.json entry")], + ), + ( + "archived on GitHub but not in the registry is DRIFT", + [{"name": "Old", "full_name": "owner/Old", "archived": True}], + { + "owner": "owner", + "repos": [ + { + "name": "Old", + "url": "https://github.com/owner/Old", + "status": "cataloged", + } + ], + }, + [ + ( + "DRIFT", + "Old: archived on GitHub but registry status is 'cataloged' (reconcile to 'archived')", + ) + ], + ), + ( + "registry says archived but GitHub disagrees is DRIFT", + [{"name": "Back", "full_name": "owner/Back", "archived": False}], + { + "owner": "owner", + "repos": [ + { + "name": "Back", + "url": "https://github.com/owner/Back", + "status": "archived", + } + ], + }, + [ + ( + "DRIFT", + "Back: registry status is 'archived' but GitHub reports it unarchived " + + "(reconcile status, or verify it was not unarchived by mistake)", + ) + ], + ), + ( + "matched, not archived either side, is clean", + [{"name": "Fine", "full_name": "owner/Fine", "archived": False}], + { + "owner": "owner", + "repos": [ + { + "name": "Fine", + "url": "https://github.com/owner/Fine", + "status": "cataloged", + } + ], + }, + [], + ), + ( + "identity matching is case-insensitive", + [{"name": "MixedCase", "full_name": "Owner/MixedCase", "archived": False}], + { + "owner": "owner", + "repos": [ + { + "name": "mixedcase", + "url": "https://github.com/owner/mixedcase", + "status": "cataloged", + } + ], + }, + [], + ), + ( + "an incidental URL whitespace difference does not miss a real match", + [{"name": "Spacey", "full_name": "owner/Spacey", "archived": False}], + { + "owner": "owner", + "repos": [ + { + "name": "Spacey", + "url": " https://github.com/owner/Spacey ", + "status": "cataloged", + } + ], + }, + [], + ), + ( + "a same-named repo under a different owner is not treated as a match", + [{"name": "Shared", "full_name": "owner/Shared", "archived": False}], + { + "owner": "owner", + "repos": [ + { + "name": "Shared", + "url": "https://github.com/someone-else/Shared", + "status": "cataloged", + } + ], + }, + [("DEFECT", "owner/Shared: exists on GitHub with no registry/repos.json entry")], + ), + ] + for label, gh_repos, spec_registry, want in cases: + with self.subTest(label): + fake = lambda owner, r=gh_repos: r + with mock.patch.object(audit, "owner_repos", fake): + got = audit.membership_findings({"registry": spec_registry}) + self.assertEqual(got, want) + + +class IntentCanonicalRelCase(unittest.TestCase): + """Holds the intent_canonical_rel resolution check.""" + + def test_intent_canonical_rel(self) -> None: + canonical_cases = [ + ( + "no reference or intentRef falls back to the file's own path", + {}, + "AGENTS.md", + "AGENTS.md", + ), + ( + "an intentRef equal to the path is itself the canonical", + {"intentRef": "GOVERNANCE.md"}, + "GOVERNANCE.md", + "GOVERNANCE.md", + ), + ( + "an anchored intentRef strips the anchor and keeps the whole file", + {"intentRef": "GOVERNANCE.md#line-endings"}, + ".editorconfig", + "GOVERNANCE.md", + ), + ( + "reference wins over intentRef when the manifest sets both", + {"reference": "catalog/snippets/configs/codecov.yml", "intentRef": "WORKFLOW.md"}, + "codecov.yml", + "catalog/snippets/configs/codecov.yml", + ), + ( + "a literal '#' in reference is preserved rather than treated as an anchor", + {"reference": "docs/notes#1.md"}, + "notes.md", + "docs/notes#1.md", + ), + ( + "a literal '#' in path is preserved rather than treated as an anchor", + {}, + "docs/notes#1.md", + "docs/notes#1.md", + ), + ( + "a non-string intentRef falls back to path instead of crashing", + {"intentRef": 123}, + "AUDIT.md", + "AUDIT.md", + ), + ( + "an anchor-only intentRef falls back to path rather than an empty canonical", + {"intentRef": "#line-endings"}, + "AUDIT.md", + "AUDIT.md", + ), + ] + for label, item, path, want in canonical_cases: + with self.subTest(label): + self.assertEqual(audit.intent_canonical_rel(item, path), want) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_fidelity_honesty.py b/tests/test_fidelity_honesty.py index 51fa150c3..ae5c655af 100755 --- a/tests/test_fidelity_honesty.py +++ b/tests/test_fidelity_honesty.py @@ -34,7 +34,7 @@ class FidelityPassAlternativesCase(unittest.TestCase): """fidelity_pass passes each unit's declared alternatives into the shared reader. - The reader's own handling of alternatives is covered by spec/audit.py's --selftest, so what + The reader's own handling of alternatives is covered by tests/test_audit_helpers.py, so what this adds is the call site. Were it to stop passing them, a repository holding only an alternative would land in the unavailable bucket while that self-test still passed. From bc8b2ce35acc61cc09c415a15c9776650c923e6f Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 23 Oct 2026 14:46:57 -0700 Subject: [PATCH 02/20] Read Workflows with PyYAML and Markdown with markdown-it-py in the Audit spec/audit.py declares pyyaml and markdown-it-py in PEP 723 inline metadata. Workflow jobs, contract tokens, the python-directories input, the concurrency group, the schedule gate, and dependabot.yml's ecosystems are read from the parsed document. Headings, fenced blocks, the README title and intro, link stripping, and the License column are read from markdown-it-py tokens. The region reading the audit, carry.py, build_dist.py, and canonical_review.py share moves to spec/carried_text.py, standard library only, so those three keep running under a plain python3. workflow_reuse.py and fidelity_honesty.py import the audit, so they declare the same libraries and their command lines run through uv. --- RESYNC.md | 2 +- docs/reusable-workflows.md | 2 +- pyproject.toml | 4 +- scripts/build_dist.py | 20 +- scripts/canonical_review.py | 8 +- scripts/carry.py | 22 +- spec/audit.py | 932 +++++++++--------------------------- spec/carried_text.py | 93 ++++ spec/fidelity_honesty.py | 8 +- spec/workflow_reuse.py | 8 +- tests/requirements.txt | 1 + tests/test_audit_helpers.py | 82 ++-- tests/test_build_dist.py | 26 +- 13 files changed, 430 insertions(+), 778 deletions(-) create mode 100644 spec/carried_text.py diff --git a/RESYNC.md b/RESYNC.md index aaa9cf68b..cc4997c9f 100644 --- a/RESYNC.md +++ b/RESYNC.md @@ -54,7 +54,7 @@ Run [`AUDIT.md`][audit] end to end. Three commands cover the mechanized part, an uv run --no-project --script spec/audit.py "" # the deterministic findings, read at main uv run --no-project --script spec/audit.py --branch "" "" # the same, read at a named ref, for convergence still in flight uv run --no-project --script spec/audit.py --issue "" # audits again and renders that run as an issue ready to file, so it can differ from the line above -python3 spec/fidelity_honesty.py --report # regenerate reports/divergences.md before using it as a work list +uv run --no-project --script spec/fidelity_honesty.py --report # regenerate reports/divergences.md before using it as a work list ``` **The branch is `main`, and the registry field naming it is not a choice a resync makes.** The runner reads the repository's `groundTruthBranch`, and [`AUDIT.md`][audit] section 1 owns what that value may be: `main` is the released and gated state for **both** workflow models, so a field naming `develop` contradicts the procedure rather than configuring it. Every cataloged repository declares `main` or omits the field and takes it as the default, so the first line above reads `main` in every case today. diff --git a/docs/reusable-workflows.md b/docs/reusable-workflows.md index 5c6cc11da..285e85cd3 100644 --- a/docs/reusable-workflows.md +++ b/docs/reusable-workflows.md @@ -37,7 +37,7 @@ The audit today grades a carried workflow at `interface` fidelity, meaning it ch ## Measured State -`python3 spec/workflow_reuse.py --report` reads every cataloged repo's `.github/workflows/` from its ground-truth branch. It compares each file against the hub canonical of the same name, after the normalization the verbatim engine applies. It clusters the copies of each canonical into variants and writes [reports/workflow-reuse.md][workflow-reuse-report]. That report is the burn-down. Its fleet total names the files, the lines, the share of lines byte-identical to a canonical, and the files that already reach a hub reusable workflow or composite action. The migration is done when the copies of each standard workflow reach zero and the callers reach the number of repos that need it. +`uv run --no-project --script spec/workflow_reuse.py --report` reads every cataloged repo's `.github/workflows/` from its ground-truth branch. It compares each file against the hub canonical of the same name, after the normalization the verbatim engine applies. It clusters the copies of each canonical into variants and writes [reports/workflow-reuse.md][workflow-reuse-report]. That report is the burn-down. Its fleet total names the files, the lines, the share of lines byte-identical to a canonical, and the files that already reach a hub reusable workflow or composite action. The migration is done when the copies of each standard workflow reach zero and the callers reach the number of repos that need it. The first run, at hub `7c67328` on 2026-08-15 before the merge-bot moved, read 108 workflow files and 10,964 lines across 20 downstream repos. 5,385 of those lines, 49 percent, were byte-identical to a hub canonical, and the rest is mostly a per-repo edit of the same canonical rather than independent code. One file reaches the hub, PhotoCleaner's `validate-task.yml` consuming the `prose-gate` action. Every other copy carries its job bodies. diff --git a/pyproject.toml b/pyproject.toml index 771b2b5fd..d14362384 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,9 +16,9 @@ files = ["spec", "host-setup", "scripts", "tests"] mypy_path = [".github/actions/prose-gate", ".github/actions/python-directories", ".github/actions/repo-gate"] exclude = 'scripts/(prose_lint|repo_gate)\.py' -# The review scripts declare markdown-it-py inline, and a type check run with no environment of its own does not install it. +# The review scripts and the audit declare their libraries inline, and a type check run with no environment of its own does not install them. [[tool.mypy.overrides]] -module = ["markdown_it", "markdown_it.*"] +module = ["markdown_it", "markdown_it.*", "yaml"] ignore_missing_imports = true [tool.pyright] diff --git a/scripts/build_dist.py b/scripts/build_dist.py index 211abd4c9..db058b9af 100755 --- a/scripts/build_dist.py +++ b/scripts/build_dist.py @@ -183,7 +183,7 @@ def write_plugin_manifest(names): # Every level, so a level-one heading ends a body, though a key names level two down, since a level-one heading is a document's title rather than a section a skill would carry. _HEADING = re.compile(r"^(#{1,6})\s+(?P\S.*?)\s*$") # CommonMark's own bound for a fence, so a marker sitting in an indented code block is content here the way it is there. -# A heading is matched at any indentation instead, the way spec/audit.py and canonical_review.py read one, so the three tools split a document alike. +# A heading is matched at any indentation instead, the way spec/carried_text.py and canonical_review.py read one, so the three tools split a document alike. _MAX_INDENT = 3 @@ -191,15 +191,15 @@ def write_plugin_manifest(names): # The module scripts/skills_install.py imports this one at module scope, and host-setup/*/install-skills.* and scripts/skills_install.sh each commit to Python 3.7+, because that path runs before the fleet's own toolchain exists to install a newer interpreter. # So functools.cache here would raise AttributeError at import on 3.7 and 3.8, breaking the one path that has to survive an old host, and spec/host-tools.json's python3 entry records that 3.7 floor and why it sits below the fleet's own. @functools.lru_cache(maxsize=None) # noqa: UP033 -def _audit_module(): - """spec/audit.py, imported on first use rather than at module import, then held. +def _carried_text(): + """spec/carried_text.py, imported on first use rather than at module import, then held. Imported lazily because scripts/skills_install.py imports this module on hosts whose Python - predates what audit.py needs, and installing never fills a region. Held because the caller + predates what the spec scripts need, and installing never fills a region. Held because the caller below reaches for it once per line, and re-resolving it there costs a sys.path scan and a sys.modules lookup every time: one pass over the 24 authored skills, 4644 lines across the 40 Markdown files they comprise, measured warm in one process, best of five, costs 17.5 ms - re-resolved against 0.9 ms held, where calling audit directly costs 0.7 ms. One --check run + re-resolved against 0.9 ms held, where calling it directly costs 0.7 ms. One --check run makes 18708 of these calls, four times that corpus, because it scans the authored tree and both generated trees, and it runs on every pull request. The cache sits here rather than on _fence_step, whose arguments vary per line and would make one unbounded. @@ -207,18 +207,18 @@ def _audit_module(): spec = str(ROOT / "spec") if spec not in sys.path: sys.path.insert(0, spec) - import audit + import carried_text - return audit + return carried_text def _fence_step(line, marker, marker_len): - """spec/audit.py's fence reading. + """spec/carried_text.py's fence reading. One reading of CommonMark across the fidelity checks, the review ledger, and this generator, rather than a second one here that could disagree with them. """ - return _audit_module()._fence_step(line, marker, marker_len) + return _carried_text().fence_step(line, marker, marker_len) def _lf(text): @@ -318,7 +318,7 @@ def heading_body(lines, heading, key): """The lines under the one heading whose text case-folds to `heading`, up to the next heading at its level or above. Any level from two down, so a key can name a GOVERNANCE.md section, an AGENTS.md subsection, or - a section of a sibling skill with one vocabulary. Matched the way spec/audit.py matches a + a section of a sibling skill with one vocabulary. Matched the way spec/carried_text.py matches a declared section, on the parsed text case-folded and at any indentation, so a re-cased heading still resolves and the two split a document alike. Two matches are two answers to one question, refused rather than resolved to the first, and a heading inside a fenced code block diff --git a/scripts/canonical_review.py b/scripts/canonical_review.py index 30c3c7779..e9335310b 100755 --- a/scripts/canonical_review.py +++ b/scripts/canonical_review.py @@ -62,11 +62,11 @@ ROOT = Path(__file__).resolve().parent.parent -# `spec/audit.py` is a script rather than a package module, and it imports its own sibling by bare name, so the spec directory has to be importable before it is. +# `spec/carried_text.py` is a script-directory module rather than a package one, so the spec directory has to be importable before it is. sys.path.insert(0, str(ROOT / "spec")) -import audit import build_dist +import carried_text import carry import local_review @@ -154,7 +154,7 @@ def decode(rel: str, data: bytes) -> str: def split_markdown(text: str) -> list[tuple[str | None, str]]: """`(heading, region)` pairs: the preamble under `None`, then one region per level-two heading. - Fence state comes from `spec/audit.py`'s own step function rather than a second reading of + Fence state comes from `spec/carried_text.py`'s step function rather than a second reading of CommonMark, so a `## ` shown inside a code sample cannot split a unit here while leaving the fidelity checks reading the document the other way. A document read two ways by two tools is the failure that helper was extracted to end. @@ -166,7 +166,7 @@ def split_markdown(text: str) -> list[tuple[str | None, str]]: marker: str | None = None marker_len = 0 for line in normalize(text).split("\n"): - marker, marker_len, boundary = audit._fence_step(line, marker, marker_len) + marker, marker_len, boundary = carried_text.fence_step(line, marker, marker_len) stripped = line.strip() if not boundary and marker is None and stripped.startswith("## "): regions.append((stripped[2:].strip(), [line])) diff --git a/scripts/carry.py b/scripts/carry.py index 54569dc8f..e5612487e 100755 --- a/scripts/carry.py +++ b/scripts/carry.py @@ -390,8 +390,8 @@ def verify_target( @functools.cache -def audit_module() -> Any: - """`spec/audit.py`, imported on first use rather than at module import, then held. +def carried_text_module() -> Any: + """`spec/carried_text.py`, imported on first use rather than at module import, then held. It is a script rather than a package module and imports its own sibling by bare name, so the spec directory has to be importable before it is. Lazily, so a tree carry never pays to load @@ -403,9 +403,9 @@ def audit_module() -> Any: spec = str(ROOT / "spec") if spec not in sys.path: sys.path.insert(0, spec) - import audit + import carried_text - return audit + return carried_text # A line and its own terminator, splitting on LF alone. `str.splitlines` also breaks on a form feed and on several Unicode separators, which are content in a carried document rather than line boundaries, so splitting on it would move bytes this tool was asked to leave alone. @@ -415,7 +415,7 @@ def audit_module() -> Any: def split_lines(text: str, rel: str | None = None) -> list[str]: """`text` as lines that keep their own terminators, so joining them returns the original bytes. - A lone CR is refused rather than read as content, because `spec/audit.py`'s `normalize` treats + A lone CR is refused rather than read as content, because `spec/carried_text.py`'s `normalize` treats one as a line break and this does not, so the two would read a CR-terminated document as different line sets: the audit would report a section stale that this tool cannot even locate, and the refusal it would otherwise raise names an absent section rather than the real problem. @@ -452,7 +452,7 @@ def file_ending(lines: list[str]) -> str: def normalize_eol(text: str) -> str: """`text` with every line ending as LF. - Deliberately not `spec/audit.py`'s `normalize`, which also masks a Dependabot-owned action pin + Deliberately not `spec/carried_text.py`'s `normalize`, which also masks a Dependabot-owned action pin and a per-repository `needs:` list. Those are governed drift for a comparison asking whether two copies say the same thing, and this tool writes bytes, so masking either would let it overwrite a value the repository owns. Line endings are the one exception, because @@ -464,13 +464,13 @@ def normalize_eol(text: str) -> str: def fence_step(line: str, marker: str | None, marker_len: int) -> tuple[str | None, int, bool]: - """`spec/audit.py`'s fence reading, so one CommonMark reading spans the fidelity checks and this. + """`spec/carried_text.py`'s fence reading, so one CommonMark reading spans the fidelity checks and this. A second reading here could disagree with the audit's about where a section stops, and then a `## ` shown inside a code sample would end a region for one tool and not the other, leaving the audit reporting drift in text this tool never touched. """ - return audit_module()._fence_step(line, marker, marker_len) + return carried_text_module().fence_step(line, marker, marker_len) def h2_headings(lines: list[str]) -> list[str]: @@ -489,7 +489,7 @@ def h2_headings(lines: list[str]) -> list[str]: def section_span(lines: list[str], heading: str) -> tuple[int, int] | None: """The `[start, end)` line range of the `## ` region, or None when the file has none. - Boundaries are `spec/audit.py`'s `extract_section`: the heading line is inside the region, a + Boundaries are `spec/carried_text.py`'s `extract_section`: the heading line is inside the region, a sibling level-two heading ends it, a fenced `## ` is content, and the match is on the parsed heading text case-folded, so a re-cased heading is found rather than read as absent. The range is positional rather than the text itself, which is what lets the caller replace the region @@ -512,7 +512,7 @@ def section_span(lines: list[str], heading: str) -> tuple[int, int] | None: def comparable_region(lines: list[str], span: tuple[int, int]) -> str: - """The region rendered the way `spec/audit.py`'s `extract_section` renders it. + """The region rendered the way `spec/carried_text.py`'s `extract_section` renders it. That function splits the whole document on LF and joins the region's lines back with LF, so a file ending in a newline yields one final empty element and a region running to the end of the @@ -555,7 +555,7 @@ def guard_governed_drift(path: str, section: str, region: str, side: str) -> Non section declared verbatim carries either today, and this refuses rather than discovering it silently the first time one does. """ - if audit_module().normalize(region) != normalize_eol(region): + if carried_text_module().normalize(region) != normalize_eol(region): raise CarryError( f"{path} section '{section}' carries content the fidelity comparison normalizes away" f" ({side} copy), so re-vendoring its bytes could revert per-repository drift." diff --git a/spec/audit.py b/spec/audit.py index 434dc569b..a4cbf7576 100755 --- a/spec/audit.py +++ b/spec/audit.py @@ -1,5 +1,9 @@ #!/usr/bin/env python3 -"""Live fleet audit: the deterministic subset of AUDIT.md, run from the hub (stdlib + gh). +# /// script +# requires-python = ">=3.13" +# dependencies = ["markdown-it-py>=4", "pyyaml>=6"] +# /// +"""Live fleet audit: the deterministic subset of AUDIT.md, run from the hub (gh plus the libraries above). Compares each cataloged registry repo against the ground truth in this repo - general settings (repo-config/settings.json), branch rulesets (normalized diff vs the model's payloads), secret @@ -21,7 +25,7 @@ failed, so the repo could not be fully audited). Exits non-zero when any repo has a DEFECT, LETTER, or ERROR finding. -Usage: python3 spec/audit.py [RepoName ...] [--branch REF] (default: every cataloged repo, +Usage: uv run --no-project --script spec/audit.py [RepoName ...] [--branch REF] (default: every cataloged repo, each read at its registry groundTruthBranch). --branch overrides that branch for the run, so a convergence can be verified before it is promoted, without editing the registry. @@ -48,7 +52,10 @@ from datetime import UTC, datetime from typing import Any +import markdown_it import validate # sibling, import-safe (its main is guarded) +import yaml +from carried_text import extract_section, normalize, strip_sections from git_env import git_location_cleared ROOT = pathlib.Path(__file__).resolve().parent.parent @@ -1031,9 +1038,6 @@ def applies(applies_to, sel): return bool(set(tokens) & sel) -_HEADING = re.compile(r"^#{1,6}\s+(.*?)\s*$") - - def _section_spec(elt): """Normalize a sections[] entry to (name, appliesTo, fidelity). A bare string is appliesTo `*`, intent.""" if isinstance(elt, str): @@ -1066,59 +1070,6 @@ def verbatim_sections(item, sel): return out -def _fence_step(line, marker, marker_len): - """One line's effect on fence state `(marker, marker_len)`: returns the state after the line, and - whether the line itself is a fence boundary (opens or closes one) rather than fenced or plain content. - - Fence matching follows CommonMark. A fence marker needs at most 3 leading spaces, more reads as - content, not a boundary. A closing fence needs the same character, a run at least as long as the - opener's, and nothing but whitespace after that run. - A mismatched or shorter marker does not close it, such as a ~~~ example inside a ``` block, or a - ``` inside a longer ````. Trailing text does not close it either, since an opening fence's language - tag has no closing counterpart. A backtick fence's info string may not itself contain a backtick, - per CommonMark, so such an opener is not a boundary either. - """ - stripped = line.lstrip(" ") - indent = len(line) - len(stripped) - char = stripped[:1] - run = len(stripped) - len(stripped.lstrip(char)) if indent <= 3 and char in ("`", "~") else 0 - if marker is None: - if run >= 3 and not (char == "`" and "`" in stripped[run:]): - return char, run, True - return None, 0, False - if char == marker and run >= marker_len and not stripped[run:].strip(): - return None, 0, True - return marker, marker_len, False - - -def extract_section(text, heading): - """The `## ` H2 section including its heading line, up to the next sibling H2 or EOF, or None if absent. - - EOL-normalized to `\\n`. The match that locates the heading is by its parsed text (the text after the `## ` - marker, case-folded and stripped of surrounding whitespace), so a re-cased heading or one with extra - marker-gap whitespace is still found rather than read as a missing section - internal heading-text - whitespace must match exactly. The heading line's exact bytes are then part of the hashed region, so that - re-casing or re-spacing surfaces as drift. A nested `###` stays inside the body. A `## ` line inside a - fenced code block, per `_fence_step`, is not a boundary. So a code sample cannot truncate the region and - hide drift after it. - """ - want = heading.strip().lower() - out, capturing, marker, marker_len = [], False, None, 0 - for ln in normalize(text).split("\n"): - stripped = ln.strip() - marker, marker_len, boundary = _fence_step(ln, marker, marker_len) - if not boundary and marker is None and stripped.startswith("## "): - if capturing: - break # a sibling H2 ends the section - if stripped[2:].strip().lower() == want: # parsed heading text after the "## " marker - capturing = True - out.append(ln) # include the heading so its exact bytes are part of the hash - continue - if capturing: - out.append(ln) - return "\n".join(out) if capturing else None - - # Carried files scanned for a coordination reference (GOVERNANCE.md "Documentation Style Conventions"). TEMPLATE_REF_SCANNED = ("AGENTS.md", "GOVERNANCE.md", ".github/copilot-instructions.md") @@ -1129,33 +1080,6 @@ def extract_section(text, heading): VERSION_LITERAL = validate.VERSION_LITERAL -def strip_sections(text, names, keep_pins=False): - """`text` with each named `## ` region removed, located by position rather than by content. - - The result is normalized, which reduces an action pin to ``, unless `keep_pins` asks for the - line endings alone to be neutralized, which is what a scan for the pin's own value needs. - - Region rules match extract_section (a fenced `## ` is not a boundary, a sibling H2 ends the region), so - the two agree on where a section starts and stops. - Positional removal is the point: deleting the extracted text instead would also delete an identical - passage anywhere else in the document, including one quoted inside the prose the caller means to read. - That failure is silent and it fails open, since the removed duplicate takes its content out of the scan. - """ - want = {n.strip().lower() for n in names} - out, dropping, marker, marker_len = [], False, None, 0 - base = text.replace("\r\n", "\n").replace("\r", "\n") if keep_pins else normalize(text) - for ln in base.split("\n"): - stripped = ln.strip() - marker, marker_len, boundary = _fence_step(ln, marker, marker_len) - if not boundary and marker is None and stripped.startswith("## "): - dropping = ( - stripped[2:].strip().lower() in want - ) # a sibling H2 always ends the previous region - if not dropping: - out.append(ln) - return "\n".join(out) - - def version_literals_outside_verbatim(text, verbatim_names): """Every version literal or commit SHA in `text` outside its verbatim sections, sorted and deduplicated. @@ -1172,16 +1096,8 @@ def version_literals_outside_verbatim(text, verbatim_names): def undeclared_h2_headings(text, declared): - """Level-two headings in `text` that `declared` does not name, sorted. - - `declared` is normalized here (stripped, lowercased), not trusted pre-normalized. The contract - then holds for any caller, regardless of how its own names are cased or spaced. - Scoped to `## ` only, the section model's unit. An H1 title or a nested H3 is not itself a - section this check judges. - Fence-aware via unfenced_text. Either a heading-syntax line or a `##`-prefixed shell comment, shown - inside a fenced code sample, is not misread as a real heading. - """ - h2s = {ln[3:].strip().lower() for ln in unfenced_text(text).split("\n") if ln.startswith("## ")} + """Level-two headings in `text` that `declared` does not name, folded and sorted.""" + h2s = {h.strip().lower() for level, h, _, _ in _atx_headings(text) if level == 2} return sorted(h2s - {d.strip().lower() for d in declared}) @@ -1199,126 +1115,62 @@ def template_ref_outside_verbatim(text, verbatim_names, hub_name): return hub_name.lower() in strip_sections(text, verbatim_names).lower() +# CommonMark plus tables, with escapes kept as their own tokens so a line can be rebuilt from its source. +_MD = markdown_it.MarkdownIt("commonmark").enable("table").disable("text_join") + + +def _atx_headings(text): + """`(level, raw text, first line, last line)` for each ATX heading, the form the region reader splits on.""" + tokens = _MD.parse(text) + return [ + (int(t.tag[1]), tokens[i + 1].content, t.map[0], t.map[1]) + for i, t in enumerate(tokens) + if t.type == "heading_open" and t.markup.startswith("#") and t.map + ] + + +def references(text): + """The reference definitions markdown-it reads from a document, keyed the way it resolves a link.""" + env = {} + _MD.parse(text, env) + return env.get("references", {}) + + def heading_texts(markdown): """Lowercased heading texts in a Markdown document, for case-insensitive section-presence matching.""" - return { - m.group(1).strip().lower() - for line in markdown.splitlines() - for m in (_HEADING.match(line),) - if m - } + return {h.strip().lower() for _, h, _, _ in _atx_headings(markdown)} _HTML_COMMENT = re.compile(r"", re.DOTALL) -def _bracket_matches(text, open_char, close_char): - """Map from each open_char index in text to the index just past its balanced close_char. - - A character class like `[^\\]]*` cannot count depth, so it stops at the first close and misses a link - label carrying its own nested brackets, e.g. `[API [docs]](url)`. Counting only open_char/close_char - nesting, ignoring the other bracket type, needs one such map per bracket type rather than one pass - mixing both. Built with a single left-to-right stack pass over the whole text rather than one depth- - counting scan per open position: re-scanning from every unmatched open is what made the previous - version O(N^2) on a run of N unmatched opens. A close pops the most recently pushed - open, the same pairing a fresh depth count from that open would find, so this is one pass, not N. - An open with no closing partner, or a close with nothing open, is left out of the map, same as before. - - A backslash-escaped delimiter (`\\[`, `\\]`, `\\(`, `\\)`) is skipped rather than pushed or popped, - matching Markdown's own escaping rule, so a literal bracket inside a label does not corrupt the nesting - count. - """ - stack = [] - matches = {} - escaped = False - for i, c in enumerate(text): - if escaped: - escaped = False - elif c == "\\": - escaped = True - elif c == open_char: - stack.append(i) - elif c == close_char and stack: - matches[stack.pop()] = i + 1 - return matches - - -def markdown_link_spans(text): - """Yield (start, end, label) for each `[label](dest)` or `[label][ref]` use, brackets/parens balanced. - - Kept in sync with spec/validate.py's contains_description_markdown_link(), which needs the same - balanced-nesting rule for the same reason. - """ - bracket_close = _bracket_matches(text, "[", "]") - paren_close = _bracket_matches(text, "(", ")") - i, n = 0, len(text) - while i < n: - if text[i] != "[": - i += 1 - continue - label_end = bracket_close.get(i) - if label_end is None: - i += 1 - continue - label = text[i + 1 : label_end - 1] - dest_end = paren_close.get(label_end) if label_end < n and text[label_end] == "(" else None - if dest_end is not None: - yield i, dest_end, label - i = dest_end - continue - ref_end = bracket_close.get(label_end) if label_end < n and text[label_end] == "[" else None - if ref_end is not None: - yield i, ref_end, label - i = ref_end - continue - # This span is not itself a link. - # A nested bracket run starting inside it may still be one, e.g. `[[docs](url)]`. - # Retry one character in rather than skipping past the whole span. - i += 1 - - -def strip_md_links(text): - """Markdown links reduced to their text - `[text](url)` and `[text][ref]` become `text`. - - The plain-text form GOVERNANCE.md "Repository Details" says the About description carries. - """ - spans = list(markdown_link_spans(text)) - if not spans: - return text - out, pos = [], 0 - for start, end, label in spans: - out.append(text[pos:start]) - out.append(label) - pos = end - out.append(text[pos:]) +def strip_md_links(text, references=None): + """`text` with each Markdown link reduced to its label, everything else kept as written. + + `references` is the document's reference table, so `[text][ref]` is a link only where `ref` is defined. + """ + out = [] + for tok in _MD.parseInline(text, {"references": dict(references or {})})[0].children or []: + if tok.type in ("text", "html_inline"): + out.append(tok.content) + elif tok.type == "code_inline": + out.append(tok.markup + tok.content + tok.markup) + elif tok.type == "softbreak": + out.append("\n") + elif tok.type not in ("link_open", "link_close"): + out.append(tok.markup) return "".join(out) def title_and_intro(text): - """The H1 title text and the intro region before the first H2, HTML comments stripped. - - Drives the README/HISTORY mirror check (spec/readme-structure.md "HISTORY.md"): both files open with the - same title and intro, and the README's ToC-omit comment must not read as a difference. - """ - norm = _HTML_COMMENT.sub("", normalize(text)) - title, region, seen_h1 = None, [], False - for ln in norm.split("\n"): - s = ln.strip() - if not seen_h1: - if s.startswith("# "): - title = s[2:].strip() - seen_h1 = True - continue - if s.startswith("## "): - break - region.append(ln.rstrip()) - # Trim the blank lines surrounding the region but keep the interior ones, so a paragraph-boundary difference is a real difference. - # The spec says the intro is copied verbatim. - while region and not region[0]: - region.pop(0) - while region and not region[-1]: - region.pop() - return title, "\n".join(region) + """The H1 title text and the intro region between it and the first H2, HTML comments stripped.""" + lines = _HTML_COMMENT.sub("", normalize(text)).split("\n") + heads = _atx_headings("\n".join(lines)) + first = next((h for h in heads if h[0] == 1), None) + if first is None: + return None, "" + end = next((h[2] for h in heads if h[0] == 2 and h[2] > first[2]), len(lines)) + return first[1].strip(), "\n".join(ln.rstrip() for ln in lines[first[3] : end]).strip("\n") def tagline(intro): @@ -1332,6 +1184,7 @@ def tagline(intro): return intro.split("\n")[0] +# Text scans, since a parsed link keeps its target but not the reference name and position these checks read. _MD_IMAGE_REF = re.compile(r"!\[[^\]]*\]\[([^\]]+)\]") _MD_IMAGE_INLINE = re.compile(r"!\[[^\]]*\]\((\S+?)\)") _LINK_DEF = re.compile(r"^\[([^\]]+)\]:\s*(\S+)", re.MULTILINE) @@ -1341,20 +1194,12 @@ def tagline(intro): def unfenced_text(text): - """`text` with every fenced block removed, EOL-normalized. - - Markup shown inside a code sample is being displayed rather than used, so a `[ref]: url`, a - `` or an `![alt][ref]` in one is not a definition, a group, or a rendered badge. Kept as - one helper because the checkers were fence-aware in some places and blind in others, which is the state - that lets a document be read two ways by one audit. - Fence matching, including boundary lines themselves, is `_fence_step`'s contract. - """ - out, marker, marker_len = [], None, 0 - for ln in normalize(text).split("\n"): - marker, marker_len, boundary = _fence_step(ln, marker, marker_len) - if not boundary and marker is None: - out.append(ln) - return "\n".join(out) + """`text` normalized with every fenced code block's lines removed, since markup in a sample is shown rather than used.""" + lines = normalize(text).split("\n") + fenced = { + i for t in _MD.parse("\n".join(lines)) if t.type == "fence" and t.map for i in range(*t.map) + } + return "\n".join(ln for i, ln in enumerate(lines) if i not in fenced) def readme_region(text, heading): @@ -1407,34 +1252,20 @@ def addressed_region(text, address): body = readme_region(text, parts[0]) if body is None or len(parts) == 1: return body - want, out, capturing = parts[1].lower(), [], False - for ln in body.split("\n"): - s = ln.strip() - if s.startswith("### "): - if capturing: - break - capturing = s[4:].strip().lower() == want - continue - if capturing: - out.append(ln) - return "\n".join(out) if capturing else None + lines = body.split("\n") + h3s = [h for h in _atx_headings(body) if h[0] == 3] + for i, (_, name, _, end) in enumerate(h3s): + if name.strip().lower() == parts[1].lower(): + stop = h3s[i + 1][2] if i + 1 < len(h3s) else len(lines) + return "\n".join(lines[end:stop]) + return None def ordered_headings(markdown, level): - """Heading texts at `level` in document order, HTML comments stripped, fenced blocks skipped. - - Distinct from heading_texts, which returns an unordered set across every level: the order check needs the - sequence, and the ToC-omit comment (`## License `) must not read as a different name. - A `## ` line inside a fenced block is not a heading, matching extract_section, so a code sample cannot - inject a phantom section. - """ - marker = "#" * level + " " - out = [] - for ln in _HTML_COMMENT.sub("", unfenced_text(markdown)).split("\n"): - s = ln.strip() - if s.startswith(marker): - out.append(s[len(marker) :].strip()) - return out + """Heading texts at `level` in document order, HTML comments stripped, fenced blocks skipped.""" + return [ + _HTML_COMMENT.sub("", h).strip() for lv, h, _, _ in _atx_headings(markdown) if lv == level + ] def readme_section_findings(text, model, sel, public): @@ -1751,30 +1582,9 @@ def readme_link_findings(text, model, slug): return findings -# Both outer pipes are optional because GitHub's Markdown makes them optional, so a row written without either is still a row this section is graded on. -# Requiring them skipped such a row outright, so its link, its description and its place in the ordering all went unread and the repo scored clean on a table nothing had read. -# The separating pipe is what makes a row a row, and it is the one this keeps demanding, since a line carrying none of them is prose. +# A tool row is read as text for its reference name, with both outer pipes optional as GitHub's Markdown makes them. _TOOL_ROW = re.compile(r"^\|?\s*\[([^\]]+)\]\[([^\]]+)\]\s*\|\s*([^|]*?)\s*(?:\||$)") _TOOL_BULLET = re.compile(r"^[-*]\s*\[([^\]]+)\]\[([^\]]+)\]\s*(.*)$") -_DELIM_CELL = re.compile(r"^:?-{3,}:?$") - - -def table_cells(line): - """The cells of a Markdown table row, or None where the line is not one. - - Both outer pipes are dropped where they are there, and neither is required, for the same reason _TOOL_ROW - stopped requiring them. - - A separating pipe is required instead, which is what keeps a thematic break out: `---` alone carries no - pipe and returns None here, where treating it as a one-cell delimiter row would have read the paragraph - above it as a table header. - """ - s = line.strip() - if "|" not in s: - return None - s = s.removeprefix("|") - s = s.removesuffix("|") - return [c.strip() for c in s.split("|")] def third_party_tool_findings(text, catalog): @@ -1802,19 +1612,18 @@ def third_party_tool_findings(text, catalog): defs = {m.group(1): m.group(2) for m in _LINK_DEF.finditer(unfenced_text(text))} findings = [] listed = [] - lines = body.split("\n") - for i, ln in enumerate(lines): - cells = table_cells(ln) - if i and cells and all(_DELIM_CELL.match(c) for c in cells): - header = table_cells(lines[i - 1]) or [] - if any(c.lower() == "license" for c in header): - findings.append( - ( - "LETTER", - "readme: 3rd Party Tools carries a License column - a license belongs to the dependency and is authoritative at its source, so restating it adds a maintenance obligation and no information (spec/readme-structure.md)", - ) + in_head = False + for tok in _MD.parse(body): + if tok.type in ("thead_open", "thead_close"): + in_head = tok.type == "thead_open" + elif in_head and tok.type == "inline" and tok.content.strip().lower() == "license": + findings.append( + ( + "LETTER", + "readme: 3rd Party Tools carries a License column - a license belongs to the dependency and is authoritative at its source, so restating it adds a maintenance obligation and no information (spec/readme-structure.md)", ) - continue + ) + for ln in body.split("\n"): m = _TOOL_ROW.match(ln.strip()) or _TOOL_BULLET.match(ln.strip()) if not m: continue @@ -1983,6 +1792,7 @@ def description_findings(doc_texts, entry, live, slug): if "README.md" in doc_texts: title, intro = title_and_intro(doc_texts["README.md"]) intro_line = tagline(intro) + refs = references(doc_texts["README.md"]) # The H1 is the repository name, and a hyphenated name may render its hyphens as spaces. # Use the GitHub API's canonical name, since the registry-URL slug can carry a different case. repo_name = live.get("name") or slug.split("/")[-1] @@ -2008,14 +1818,14 @@ def description_findings(doc_texts, entry, live, slug): ) ) else: - if strip_md_links(intro_line) != intro_line: + if strip_md_links(intro_line, refs) != intro_line: findings.append( ( "LETTER", "readme: the tagline carries Markdown links - keep it link-free plain text, it doubles as the repo About description (spec/readme-structure.md)", ) ) - readme_want = strip_md_links(intro_line).strip() + readme_want = strip_md_links(intro_line, refs).strip() if len(readme_want) > 100: findings.append( ( @@ -2083,152 +1893,98 @@ def workspace_cspell_words(text): ) -_JOB_KEY = re.compile(r"^([A-Za-z0-9_.\-]+):(\s.*)?$") - +def _compose(text): + """The YAML node tree of `text`, or None where it does not parse.""" + try: + return yaml.compose(text, Loader=yaml.SafeLoader) + except yaml.YAMLError: + return None -def split_jobs(text): - """Slice a workflow YAML into {job_key: block_text} by the jobs-level indent, without a YAML parser. - Structural, not semantic: find the top-level `jobs:` key, take the indent of its first child as the - job-key indent, and cut the region at each key line at exactly that indent. A line dedented below the - job-key indent (a sibling top-level key) ends the jobs region. - """ - lines = text.splitlines(keepends=True) - ji = next((i for i, ln in enumerate(lines) if re.match(r"^jobs:\s*(#.*)?$", ln)), None) - if ji is None: - return {} - job_indent = None - for ln in lines[ji + 1 :]: - if ln.strip() == "" or ln.lstrip().startswith("#"): - continue - job_indent = len(ln) - len(ln.lstrip()) - break - if not job_indent: +def _mapping(node): + """A mapping node's children as `{key: node}`, `{}` for any other node.""" + if not isinstance(node, yaml.MappingNode): return {} - blocks, key, cur = {}, None, [] - for ln in lines[ji + 1 :]: - indent = len(ln) - len(ln.lstrip()) - if ln.strip() and indent < job_indent: - if ln.lstrip().startswith("#"): - continue # a dedented comment is not part of any job and does not end the mapping - skip it - break # a sibling top-level key ends the jobs region - m = _JOB_KEY.match(ln[job_indent:]) if indent == job_indent else None - if m: - if key is not None: - blocks[key] = "".join(cur) - key, cur = ( - m.group(1), - [ln], - ) # include the key line so an inline mapping on it is captured - elif key is not None: - cur.append(ln) - if key is not None: - blocks[key] = "".join(cur) - return blocks - - -# A `key: |`/`key: >` block-scalar indicator, with an optional anchor (`&name`) before it. -# Anchored whole-line so `key: "a|b"` does not match. -_BLOCK_SCALAR_KEY = re.compile(r"^[^:#\n]*:\s*(&\S+\s+)?[|>][0-9+-]*\s*(#.*)?$") -# A bare `- |`/`- >` sequence item, a block scalar with no mapping key at all (a matrix string). -_BARE_BLOCK_SCALAR = re.compile(r"^(&\S+\s+)?[|>][0-9+-]*\s*(#.*)?$") -# A step's `- key: |` sequence-item prefix, stripped before matching the two patterns above. -_SEQUENCE_ITEM_PREFIX = re.compile(r"^-[ \t]+") -_IF_KEY = re.compile(r"^if\s*:") - - -def _code_view(text): - """Workflow text with comment-only lines and block-scalar bodies dropped, neither is structure. - - A carried task file documents its own contract in comments (build-release-task.yml names - `release-asset-` and `artifact-ids:` in prose), so a raw substring search over the whole text would - both false-pass a missing handoff and false-flag a forbidden token that appears only in a comment. - A block-scalar string value (`name: |` followed by indented text) can hide or fake a token the same - way, so its body is dropped too, keeping only the `key:` line itself. - A block-scalar `if:` body, the form the Workflow YAML convention sets for a multi-line condition, is kept, - since an `if:` value is always an expression. - It is folded onto the key line, so a token the condition wraps across two lines still matches whole. - A trailing comment on that key line is dropped first, so it never reads as part of the condition. - """ - out = [] - skip_indent = None - fold_indent = None - for ln in text.splitlines(): - if ln.lstrip().startswith("#"): - continue - if fold_indent is not None: - if not ln.strip(): - continue - if (len(ln) - len(ln.lstrip())) > fold_indent: - out[-1] += " " + ln.strip() - continue - fold_indent = None - if skip_indent is not None: - if ln.strip() and (len(ln) - len(ln.lstrip())) <= skip_indent: - skip_indent = None # a dedent back to (or past) the key column ends the block body - else: - continue # still inside the block scalar body, not real structure - stripped = ln.lstrip() - dash_col = len(ln) - len(stripped) - key_col = dash_col - dash = _SEQUENCE_ITEM_PREFIX.match(stripped) - if dash: - # A keyed scalar's boundary is the key's own column, not the dash's. - # A sibling key genuinely at that column (`env:` alongside a `- run: |` step) is real structure. - key_col += dash.end() - stripped = stripped[dash.end() :] - if _BLOCK_SCALAR_KEY.match(stripped): - if _IF_KEY.match(stripped): - fold_indent = key_col - ln = re.sub(r"[ \t]+#.*$", "", ln) - else: - skip_indent = key_col - elif dash and _BARE_BLOCK_SCALAR.match(stripped): - # A bare `- |` has no key, so its own boundary is the dash's column, not a key past it. - skip_indent = dash_col - out.append(ln) - return "\n".join(out) + return {str(k.value): v for k, v in node.value if isinstance(k, yaml.ScalarNode)} + +def _plain(node): + """The Python value a node constructs, as the safe loader reads it.""" + return yaml.SafeLoader("").construct_object(node, deep=True) -def job_level_names(blocks): - """The job-level `name:` values - a job's own direct-child name key, not a step's name. - The ruleset-bound check is a job name, so a step happening to share the string must not satisfy it. A - job's direct children sit at the shallowest indent of its block (below the key line); a step name is - deeper or a `- name:` list item, so neither reaches that indent as a bare `name:`. +def job_nodes(text): + """The workflow's jobs as `{key: node}`, `{}` where it has none or does not parse.""" + return _mapping(_mapping(_compose(text)).get("jobs")) + + +def code_view(node, depth=0): + """A node's keys and scalar values as lines indented two spaces per level, the shape a contract token is read in. + + A job's children sit at `depth` 2, its place in a workflow. A block-scalar body is a script or prose rather + than structure, so it is left out, except an `if:` condition, folded onto one line. """ - out = set() - for block in blocks.values(): - for ln in job_child_lines(block): - m = re.match(r"name:\s*['\"]?(.*?)['\"]?\s*$", ln) - if m: - out.add(m.group(1)) - return out + out = [] + def walk(child, key, level): + label = " " * level + ("" if key is None else f"{key}:") + if isinstance(child, yaml.MappingNode): + out.extend([label] if key else []) + for k, v in child.value: + walk(v, k.value, level + 1) + elif isinstance(child, yaml.SequenceNode): + out.extend([label] if key else []) + for v in child.value: + walk(v, None, level + 1) + elif key == "if": + out.append(f"{label} {' '.join(str(child.value).split())}") + elif child.style in ("|", ">"): + out.extend([label] if key else []) + else: + out.append(f"{label} {child.value}".rstrip()) -def job_child_lines(block): - """A job block's direct-child lines, dedented, read at the shallowest indent below its key line.""" - body = [ln for ln in block.splitlines()[1:] if ln.strip() and not ln.lstrip().startswith("#")] - if not body: - return [] - child = min(len(ln) - len(ln.lstrip()) for ln in body) - return [ln[child:] for ln in body if len(ln) - len(ln.lstrip()) == child] + if node is not None: + walk(node, None, depth - 1) + return "\n".join(out) -def check_interface(path, contract, text): - """Verify a workflow honors its fixed contract by name and wiring, never its body (spec/fidelity-model.md). +def split_jobs(text): + """Each job's own source lines, `{key: text}`, cut at the parser's marks for the verbatim job hash. - All findings are DRIFT: the interface is what a repo must keep, the body is owned, and a rename or a - forked handoff is a hint to verify, not a hard failure. `verbatimJobs` is the verbatim engine's concern. + A comment dedented below the job keys belongs to no job, so it is left out of every region. """ + jobs = _mapping(_compose(text)).get("jobs") + if not isinstance(jobs, yaml.MappingNode) or jobs.flow_style or not jobs.value: + return {} + lines = text.splitlines(keepends=True) + indent = jobs.value[0][0].start_mark.column + end = jobs.end_mark.line + (1 if jobs.end_mark.column else 0) + starts = [k.start_mark.line for k, _ in jobs.value] + [end] + return { + str(k.value): "".join( + ln + for ln in lines[starts[i] : starts[i + 1]] + if not (ln.lstrip().startswith("#") and len(ln) - len(ln.lstrip()) < indent) + ) + for i, (k, _) in enumerate(jobs.value) + } + + +def check_interface(path, contract, text): + """DRIFT where a workflow breaks its fixed contract by name and wiring, never its body (spec/fidelity-model.md).""" + try: + yaml.compose(text, Loader=yaml.SafeLoader) + except yaml.YAMLError: + return [("DRIFT", f"interface: {path} does not parse as YAML, so its contract is unread")] findings = [] - jobs = split_jobs(text) - code = _code_view(text) + jobs = job_nodes(text) for k in contract.get("requiredJobKeys", []): if k not in jobs: findings.append(("DRIFT", f"interface: {path} missing required job '{k}'")) name = contract.get("requiredCheckName") - if name and name not in job_level_names(jobs): + nodes = (_mapping(job).get("name") for job in jobs.values()) + names = {n.value for n in nodes if isinstance(n, yaml.ScalarNode)} + if name and name not in names: findings.append( ( "DRIFT", @@ -2236,111 +1992,39 @@ def check_interface(path, contract, text): ) ) tok = contract.get("artifactNameToken") - if tok and tok not in code: + if tok and tok not in code_view(_compose(text)): findings.append( ("DRIFT", f"interface: {path} missing the '{tok}-' artifact handoff") ) - # Token checks only apply to a job that is present, since an absent job is already reported by requiredJobKeys, so skip it rather than emit a redundant "missing token" for every token it cannot contain. - # Scan the job's code view so a token in a comment is not read as signal. + # An absent job is already a requiredJobKeys finding, so its tokens are not reported again. for job, toks in contract.get("requireTokensInJob", {}).items(): if job in jobs: - block = _code_view(jobs[job]) + view = code_view(jobs[job], 2) for t in toks: - if t not in block: + if t not in view: findings.append( ("DRIFT", f"interface: {path} job '{job}' missing required '{t}'") ) for job, toks in contract.get("forbidTokensInJob", {}).items(): if job in jobs: - block = _code_view(jobs[job]) + view = code_view(jobs[job], 2) for t in toks: - if t in block: - # Generic rather than tied to one contract's reason. - # forbidTokensInJob guards more than the github-release seam. - # intentRef on the file's own entry routes a reader to the actual context. + if t in view: findings.append( ("DRIFT", f"interface: {path} job '{job}' uses forbidden '{t}'") ) return findings -# The two workflow files whose validate job may declare validate-task.yml's `python-directories` input, -# per WORKFLOW.md's caller contract. A stub that has not yet adopted the reusable gate names no such job -# and is skipped by python_directories_caller_findings() below rather than compared. +# The callers whose validate job may declare validate-task.yml's `python-directories` input, per WORKFLOW.md. PYTHON_DIRECTORIES_CALLERS = ( ".github/workflows/test-pull-request.yml", ".github/workflows/publish-release.yml", ) -# A `key: |`/`key: >` block-scalar indicator on a `with:` input line, matching _BLOCK_SCALAR_KEY's shape -# below but anchored to the key rather than to a whole line, since workflow_input_text() has already -# sliced the line down to the value past `key:`. -_WITH_INPUT_BLOCK_SCALAR = re.compile(r"^(&\S+[ \t]+)?[|>][0-9+-]*[ \t]*(#.*)?$") -_WITH_KEY = re.compile(r"^([ \t]*)with:[ \t]*(#.*)?$") - - -def workflow_input_text(job_text, key, raw=False): - """The raw text of a `with:` input on a workflow-call job: a block scalar's dedented body, or a plain - single value's own line, whichever shape the input was written in. With raw, the text after the key's - colon and every line nested under it instead, unprocessed, for a caller judging the value's shape. - - Structural, like split_jobs() and _code_view() above: it reads indentation, not YAML semantics, since - a hand-written `with:` block stays inside this narrow shape everywhere the fleet writes one. Returns - None where the job carries no `with:` mapping, or no such key inside it. - """ - lines = job_text.splitlines() - with_at = next((i for i, ln in enumerate(lines) if _WITH_KEY.match(ln)), None) - if with_at is None: - return None - with_indent = len(lines[with_at]) - len(lines[with_at].lstrip()) - child_indent = None - key_at = None - for i in range(with_at + 1, len(lines)): - ln = lines[i] - if not ln.strip() or ln.lstrip().startswith("#"): - continue - indent = len(ln) - len(ln.lstrip()) - if indent <= with_indent: - break # dedent out of the with: mapping - if child_indent is None: - child_indent = indent - if indent == child_indent and re.match(rf"^{re.escape(key)}:", ln[child_indent:]): - key_at = i - break - if key_at is None or child_indent is None: - return None - rest = lines[key_at][child_indent + len(key) + 1 :].strip() - if raw: - nested = [] - for ln in lines[key_at + 1 :]: - if ( - ln.strip() - and not ln.lstrip().startswith("#") - and len(ln) - len(ln.lstrip()) <= child_indent - ): - break - nested.append(ln) - return "\n".join([rest, *nested]) - if not rest or _WITH_INPUT_BLOCK_SCALAR.match(rest): - body = [] - for ln in lines[key_at + 1 :]: - if not ln.strip(): - body.append("") - continue - indent = len(ln) - len(ln.lstrip()) - if indent <= child_indent: - break # dedent back to (or past) the key column ends the block body - body.append(ln[child_indent:]) - return "\n".join(body) - return re.sub(r"[ \t]+#.*$", "", rest).strip().strip("'\"") def normalize_declared_python_directory(line): - """Mirror python_directories.py's normalize(), success case only. - - The audit's cross-check only needs the directories a caller's declaration resolves to; a malformed - line is already refused by the hub validator itself at merge time, so this drops one silently rather - than reporting the same defect a second time. - """ + """Mirror python_directories.py's normalize() for a valid line, None for one the hub validator refuses.""" path = line.strip() if not path or "\t" in path or path.startswith("/") or "\\" in path or ".." in path.split("/"): return None @@ -2348,27 +2032,20 @@ def normalize_declared_python_directory(line): def python_directories_caller_findings(path, text, entry): - """DRIFT when a caller's declared `python-directories` input disagrees with the registry's own. + """DRIFT where a caller's declared `python-directories` input disagrees with the registry's own. - Each side is normalized and deduplicated, then compared as a set of declarations rather than by - the directories each resolves to, since naming `.` is not the same as naming nothing: a declared - directory owes a suite and a type check that the root default only runs where it finds them. - - Scoped to PYTHON_DIRECTORIES_CALLERS, and only where the job's own code (comments excluded, per - _code_view()) actually names validate-task.yml, since a caller with no validate job, or one that has - not adopted the reusable gate yet, states nothing this can compare against; that absence is already - check_interface()'s finding, not this one's. A shape this cannot read is reported rather than skipped - or misread: validate-task.yml called from a job under another key, and a flow-mapping `with:`. + Compared as normalized sets of declarations, since naming `.` is not the same as naming nothing. + Only the job keyed `validate` is compared, and validate-task.yml called from another job is reported. """ if path not in PYTHON_DIRECTORIES_CALLERS: return [] - jobs = split_jobs(text) - validate_job = jobs.get("validate", "") + jobs = job_nodes(text) others = sorted( - k for k, body in jobs.items() if k != "validate" and "validate-task.yml" in _code_view(body) + k for k, job in jobs.items() if k != "validate" and "validate-task.yml" in code_view(job) ) - unread = ( - [ + unread = [] + if others: + unread = [ ( "DRIFT", ( @@ -2378,54 +2055,13 @@ def python_directories_caller_findings(path, text, entry): ), ) ] - if others - else [] - ) - if "validate-task.yml" not in _code_view(validate_job): + validate_job = jobs.get("validate") + if validate_job is None or "validate-task.yml" not in code_view(validate_job): return unread - if re.search(r"^[ \t]*with:[ \t]*\{", validate_job, re.MULTILINE): - return unread + [ - ( - "DRIFT", - ( - f"python-directories: {path} writes its validate job's with: inputs as a flow " - "mapping, which the audit cannot read. Write a block mapping, one input per line." - ), - ) - ] - value = re.sub( - r"^(?:\s|[&!]\S*|#[^\n]*)*", - "", - workflow_input_text(validate_job, "python-directories", raw=True) or "", - ) - # A folded scalar's value depends on YAML's indentation rules, so it is refused rather than guessed. - if value.startswith(">"): - return unread + [ - ( - "DRIFT", - ( - f"python-directories: {path} passes a folded (>) scalar. Write it as a literal " - "(|) block, one directory per line." - ), - ) - ] - if value.startswith('"') and "\\" in value[1:].split('"', 1)[0]: - return unread + [ - ( - "DRIFT", - ( - f"python-directories: {path} passes a double-quoted value carrying a backslash " - "escape. Write it unquoted, or as a literal (|) block, one directory per line." - ), - ) - ] - declared_text = workflow_input_text(validate_job, "python-directories") or "" + node = _mapping(_mapping(validate_job).get("with")).get("python-directories") + declared_text = "" if node is None else _scalar_text(_plain(node)) or "" declared = sorted( - { - d - for d in (normalize_declared_python_directory(ln) for ln in declared_text.splitlines()) - if d - } + {d for d in map(normalize_declared_python_directory, declared_text.splitlines()) if d} ) registry = sorted(python_directories_of(entry)) if declared == registry: @@ -2454,25 +2090,11 @@ def python_directories_caller_findings(path, text, entry): ) _SCHEDULE_RUN_ID_SUFFIX = "${{github.event_name=='schedule'&&format('-{0}',github.run_id)||''}}" _DISPATCH_TEST = "github.event_name=='workflow_dispatch'" -_TOP_CONCURRENCY = re.compile(r"^concurrency:[ \t]*(.*)$", re.MULTILINE) -_TOP_ON = re.compile(r"^on:[ \t]*(.*)$", re.MULTILINE) _EXPRESSION_SPACE = re.compile(r"\$\{\{\s*(.*?)\s*\}\}") -def _yaml_scalar(value): - """A plain or quoted scalar's text, trailing comment dropped, or None where it cannot be read as one.""" - value = value.strip() - quoted = re.match(r"""^(?:'((?:[^']|'')*)'|"((?:[^"\\]|\\.)*)")\s*(?:#.*)?$""", value) - if quoted: - single = quoted.group(1) - return single.replace("''", "'") if single is not None else quoted.group(2) - if value[:1] in ("'", '"', ">", "|"): - return None - return re.sub(r"[ \t]+#.*$", "", value) - - def _strip_unquoted_space(value): - """Whitespace removed except inside single-quoted strings, so a name such as `'a b'` stays distinct from `'ab'`.""" + """Whitespace removed except inside single-quoted strings, so `'a b'` stays distinct from `'ab'`.""" parts = value.split("'") return "'".join( re.sub(r"\s+", "", part) if i % 2 == 0 else part for i, part in enumerate(parts) @@ -2483,57 +2105,31 @@ def _tight(value): return _EXPRESSION_SPACE.sub(lambda m: "${{" + m.group(1) + "}}", value) -def _workflow_triggers(code): - """The event names keyed under a block-mapping workflow-level `on:`, or None for any other shape. - - Only the block-mapping form is read, the shape a `schedule` trigger with its `cron` list takes in practice. - """ - head = _TOP_ON.search(code) - if head is None or re.sub(r"(?:^|[ \t]+)#.*$", "", head.group(1)).strip(): +def _scalar_text(value): + """A YAML scalar as the string GitHub reads, or None for a mapping or a sequence.""" + if isinstance(value, bool): + return str(value).lower() + if isinstance(value, (dict, list)): return None - events = set() - child_indent = None - for line in code[head.end() :].splitlines()[1:]: - if not line.strip(): - continue - indent = len(line) - len(line.lstrip()) - if indent == 0: - break - if child_indent is None: - child_indent = indent - if indent != child_indent: - continue - m = re.match(r"^['\"]?([A-Za-z_]+)['\"]?[ \t]*:", line.strip()) - if m is None: - return None - events.add(m.group(1)) - return events + return "" if value is None else str(value) -def _schedule_never_publishes(code): - """True where the publisher's `on:` admits a `schedule` and no `push`, and every release job requires a dispatch. +def _schedule_never_publishes(doc): + """True where `on:` admits a `schedule` and no `push`, and every build-release-task.yml job requires a dispatch. - The plan task reports a schedule as a publish. A schedule therefore publishes nothing only where each job - calling `build-release-task.yml` holds `github.event_name == 'workflow_dispatch'` as a top-level conjunct - of its job-level `if:`. - A condition holding any `||` or a negated group is not read as such a gate. A publisher where no job - calls `build-release-task.yml` returns False. + The dispatch test must be a top-level conjunct of the job's `if:`, in a condition holding no `||` or negated group. """ - triggers = _workflow_triggers(code) - if triggers is None or "schedule" not in triggers or "push" in triggers: + # PyYAML reads a bare `on` key as the boolean true, per YAML 1.1. + on = doc.get("on", doc.get(True)) + triggers = set(on) if isinstance(on, (dict, list)) else {on} + if "schedule" not in triggers or "push" in triggers: return False release_jobs = 0 - for block in split_jobs(code).values(): - keys = {} - for ln in job_child_lines(block): - m = re.match(r"^(uses|if):[ \t]*(.*)$", ln) - if m: - keys[m.group(1)] = m.group(2) - if "build-release-task.yml" not in keys.get("uses", ""): + for job in (doc.get("jobs") or {}).values(): + if not isinstance(job, dict) or "build-release-task.yml" not in str(job.get("uses", "")): continue release_jobs += 1 - condition = _yaml_scalar(re.sub(r"^[|>][0-9+-]*[ \t]*", "", keys.get("if", ""))) - tight = _strip_unquoted_space(condition or "") + tight = _strip_unquoted_space(_scalar_text(job.get("if")) or "") if tight.startswith("${{") and tight.endswith("}}"): tight = tight[3:-2] conjuncts = {term.strip("()") for term in tight.split("&&")} @@ -2543,15 +2139,11 @@ def _schedule_never_publishes(code): def concurrency_caller_findings(path, text): - """DRIFT when the publisher or the merge-bot declares a concurrency group other than the one WORKFLOW.md pins. + """DRIFT where the publisher or the merge-bot declares a concurrency group other than the one WORKFLOW.md pins. - The publisher's group is the ref-scoped base, optionally followed by one `${{ }}` expression naming both - `github.event_name` and `github.run_id`. That is the push-only suffix, or the schedule-only suffix - on a publisher whose schedule never publishes. The merge-bot's group is the - base keyed on the PR number. Either must leave `cancel-in-progress` false or unset. Only the - workflow-level `concurrency:` is read, as a block mapping or a bare group scalar. A shape this cannot - read is reported rather than skipped. A file absent from the repo never reaches here, since the - interface unit reports that. + The publisher's group is the ref-scoped base plus at most the push-only run-id suffix, or the schedule-only + suffix where its schedule never publishes. The merge-bot's is the base keyed on the PR number. Either leaves + `cancel-in-progress` false or unset. """ base = CONCURRENCY_CALLERS.get(path) if base is None: @@ -2562,45 +2154,25 @@ def concurrency_caller_findings(path, text): f"concurrency: {path} writes its concurrency in a shape the audit cannot read. Write a block mapping with group and cancel-in-progress, each a single-line scalar.", ) ] - code = _code_view(text) - head = _TOP_CONCURRENCY.search(code) - if head is None: + try: + doc = yaml.safe_load(text) + except yaml.YAMLError: + return unreadable + if not isinstance(doc, dict): + return unreadable + concurrency = doc.get("concurrency") + if concurrency is None: return [ ( "DRIFT", f"concurrency: {path} declares no workflow-level concurrency, expected group '{base}'", ) ] - group = cancel = None - inline = re.sub(r"^(?:&\S+[ \t]*)?(?:#.*)?", "", head.group(1)).strip() - if inline.startswith("{"): + raw_cancel = concurrency.get("cancel-in-progress") if isinstance(concurrency, dict) else None + group = _scalar_text(concurrency.get("group") if isinstance(concurrency, dict) else concurrency) + cancel = None if raw_cancel is None else _scalar_text(raw_cancel) + if group is None or (raw_cancel is not None and cancel is None): return unreadable - if inline: - group = _yaml_scalar(inline) - following = next((ln for ln in code[head.end() :].splitlines()[1:] if ln.strip()), "") - if group is None or following[:1] in (" ", "\t"): - return unreadable - else: - child_indent = None - for line in code[head.end() :].splitlines()[1:]: - if not line.strip(): - continue - indent = len(line) - len(line.lstrip()) - if indent == 0: - break - if child_indent is None: - child_indent = indent - if indent > child_indent: - return unreadable - m = re.match(r"^[ \t]+(group|cancel-in-progress):[ \t]*(.*)$", line) - if m: - value = _yaml_scalar(m.group(2)) - if value is None: - return unreadable - if m.group(1) == "group": - group = value - else: - cancel = value findings = [] if not group: findings.append(("DRIFT", f"concurrency: {path} reads no group value, expected '{base}'")) @@ -2608,25 +2180,18 @@ def concurrency_caller_findings(path, text): tight_group, tight_base = _tight(group), _tight(base) suffix = tight_group[len(tight_base) :] if tight_group.startswith(tight_base) else None tight_suffix = _strip_unquoted_space(suffix) if suffix else "" + publisher = path.endswith("publish-release.yml") + schedule_suffix = publisher and tight_suffix == _SCHEDULE_RUN_ID_SUFFIX suffix_ok = suffix is not None and ( suffix == "" - or ( - path.endswith("publish-release.yml") - and ( - bool(_RUN_ID_SUFFIX.match(tight_suffix)) - or (tight_suffix == _SCHEDULE_RUN_ID_SUFFIX and _schedule_never_publishes(code)) - ) - ) + or (publisher and bool(_RUN_ID_SUFFIX.match(tight_suffix))) + or (schedule_suffix and _schedule_never_publishes(doc)) ) - if ( - not suffix_ok - and path.endswith("publish-release.yml") - and tight_suffix == _SCHEDULE_RUN_ID_SUFFIX - ): + if not suffix_ok and schedule_suffix: findings.append( ( "DRIFT", - f"concurrency: {path} gives a schedule its own group, but the audit cannot read its schedule as never publishing. That needs a block-mapping on: with no push trigger, and github.event_name == 'workflow_dispatch' as a top-level conjunct of each build-release-task.yml job's if:, an if: holding no || or negated group.", + f"concurrency: {path} gives a schedule its own group, but the audit cannot read its schedule as never publishing. That needs an on: with no push trigger, and github.event_name == 'workflow_dispatch' as a top-level conjunct of each build-release-task.yml job's if:, an if: holding no || or negated group.", ) ) elif not suffix_ok: @@ -2638,34 +2203,20 @@ def concurrency_caller_findings(path, text): return findings -# A `uses: @<40-hex sha>` pin, plus only a trailing Dependabot version comment such as ` # v1.2.3`, where the leading `v` or digit is required. -# Dependabot bumps both per repo, so that drift is governed the way EOL is rather than being a fidelity deviation. -# It is anchored to `uses:`, so a 64-hex docker digest and a tag or branch ref such as `@v4` do not match. -# Hex is case-insensitive, and a hand-written note on a pin is not version-shaped, so it survives to be compared. -_ACTION_PIN = re.compile(r"(\buses:[ \t]*[^\s@]+)@[0-9a-fA-F]{40}(?:[ \t]+#[ \t]*v?[0-9][\w.\-]*)?") -# A workflow job's `needs:` list names the jobs it sequences after. -# In a verbatim job region a repo prunes that list to the targets it actually vendors, since a `needs` entry naming an unvendored job fails the whole workflow to load, so the list is owned per repo rather than fixed. -# Mask it in the inline, scalar and block-list forms, the same as the action pin. -# The interface contract still checks the required job keys separately. -_JOB_NEEDS = re.compile( - r"(^[ \t]*)needs:[ \t]*" - r"(?:\[[^\]\n]*\]" # inline: needs: [a, b] (same line only) - r"|[A-Za-z0-9_.\-]+[ \t]*(?=\n|$)" # scalar: needs: a - r"|(?:\n[ \t]+-[ \t]*[A-Za-z0-9_.\-]+[ \t]*)+)", # block: needs:\n - a\n - b - re.MULTILINE, -) - - -def normalize(text): - """Reduce a carried unit to its comparable form: neutralize line endings (EOL variance is governed - separately, not a fidelity deviation), a Dependabot-owned action pin (the 40-hex `uses: @` commit - plus its ` # vN` comment, bumped per repo), and a job's owned `needs:` list (pruned per repo to its vendored - targets). All three are governed drift, not a fidelity deviation. This is NOT placeholder masking of declared - per-file tokens; see spec/fidelity-model.md "Normalization". - """ - text = text.replace("\r\n", "\n").replace("\r", "\n") - text = _ACTION_PIN.sub(r"\1@", text) - return _JOB_NEEDS.sub(r"\1needs: ", text) +def dependabot_ecosystems(text): + """The package-ecosystem values a dependabot.yml declares, empty where it does not parse.""" + try: + doc = yaml.safe_load(text) + except yaml.YAMLError: + return set() + updates = doc.get("updates") if isinstance(doc, dict) else None + if not isinstance(updates, list): + return set() + return { + str(u["package-ecosystem"]) + for u in updates + if isinstance(u, dict) and u.get("package-ecosystem") + } @functools.lru_cache( @@ -3363,7 +2914,6 @@ def read_text(path): # --- Dependabot ecosystem coverage --- # A repo's tree implies Dependabot ecosystems it must track, being github-actions where it ships workflows and devcontainers where it ships a .devcontainer. # Without the first, the action versions those workflows reference go stale and a merge-bot then has no PRs to auto-merge. - # The dependabot.yml file is YAML and no stdlib parser reads it, so scan the declared package-ecosystem values by regex, anchored to the line start so a commented-out entry is not read as declared. # This asserts an implied ecosystem's *presence* only. # Whether each declared ecosystem dual-targets main and develop, which is the fleet norm, is verified by inspection rather than here. # It only runs where dependabot.yml exists, since its absence is already a file-presence LETTER below. @@ -3371,13 +2921,7 @@ def read_text(path): db = gh(f"repos/{slug}/contents/.github/dependabot.yml?ref={ground}", ok404=True) db_text = contents_text(db) if db_text: - declared = set( - re.findall( - r'^[ \t]*-?[ \t]*package-ecosystem:[ \t]*["\']?([\w-]+)', - db_text, - re.MULTILINE, - ) - ) + declared = dependabot_ecosystems(db_text) implied = {} workflows = gh(f"repos/{slug}/contents/.github/workflows?ref={ground}", ok404=True) if isinstance(workflows, list) and any( diff --git a/spec/carried_text.py b/spec/carried_text.py new file mode 100644 index 000000000..9a537e4e1 --- /dev/null +++ b/spec/carried_text.py @@ -0,0 +1,93 @@ +"""The region reading the fidelity tools share, so each one splits a document alike. + +Standard library only, because `scripts/build_dist.py`, `scripts/carry.py`, and +`scripts/canonical_review.py` import it under a plain `python3`. +""" + +import re + +# A `uses: @<40-hex sha>` pin plus an optional version-shaped comment, which Dependabot bumps per repo. +_ACTION_PIN = re.compile(r"(\buses:[ \t]*[^\s@]+)@[0-9a-fA-F]{40}(?:[ \t]+#[ \t]*v?[0-9][\w.\-]*)?") +# A job's `needs:` list, inline, scalar, or block, which a repo prunes to the targets it vendors. +_JOB_NEEDS = re.compile( + r"(^[ \t]*)needs:[ \t]*" + r"(?:\[[^\]\n]*\]" + r"|[A-Za-z0-9_.\-]+[ \t]*(?=\n|$)" + r"|(?:\n[ \t]+-[ \t]*[A-Za-z0-9_.\-]+[ \t]*)+)", + re.MULTILINE, +) + + +def normalize(text): + """`text` with line endings, action pins, and job `needs:` lists neutralized, the drift governed elsewhere. + + This is not placeholder masking, see spec/fidelity-model.md "Normalization". + """ + text = text.replace("\r\n", "\n").replace("\r", "\n") + text = _ACTION_PIN.sub(r"\1@", text) + return _JOB_NEEDS.sub(r"\1needs: ", text) + + +def fence_step(line, marker, marker_len): + """One line's effect on fence state `(marker, marker_len)`, as `(marker, marker_len, is_boundary)`. + + CommonMark fences: at most 3 leading spaces, a closer of the opener's character at least as long + with nothing after it, and no backtick in a backtick fence's info string. + """ + stripped = line.lstrip(" ") + indent = len(line) - len(stripped) + char = stripped[:1] + run = len(stripped) - len(stripped.lstrip(char)) if indent <= 3 and char in ("`", "~") else 0 + if marker is None: + if run >= 3 and not (char == "`" and "`" in stripped[run:]): + return char, run, True + return None, 0, False + if char == marker and run >= marker_len and not stripped[run:].strip(): + return None, 0, True + return marker, marker_len, False + + +def _h2_text(line, marker, boundary): + """The folded text of a `## ` heading outside a fence, or None for any other line.""" + stripped = line.strip() + if boundary or marker is not None or not stripped.startswith("## "): + return None + return stripped[2:].strip().lower() + + +def extract_section(text, heading): + """The normalized `## ` region, heading line included, up to the next H2, or None. + + The heading is found by its folded text, and its exact bytes are part of the region, so a re-cased + heading is found and still reads as drift. + """ + want = heading.strip().lower() + out, capturing, marker, marker_len = [], False, None, 0 + for ln in normalize(text).split("\n"): + marker, marker_len, boundary = fence_step(ln, marker, marker_len) + name = _h2_text(ln, marker, boundary) + if name is not None: + if capturing: + break + capturing = name == want + if capturing: + out.append(ln) + return "\n".join(out) if capturing else None + + +def strip_sections(text, names, keep_pins=False): + """`text` with each named `## ` region removed by position, normalized unless `keep_pins`. + + Positional, so an identical passage elsewhere in the document survives. + """ + want = {n.strip().lower() for n in names} + out, dropping, marker, marker_len = [], False, None, 0 + base = text.replace("\r\n", "\n").replace("\r", "\n") if keep_pins else normalize(text) + for ln in base.split("\n"): + marker, marker_len, boundary = fence_step(ln, marker, marker_len) + name = _h2_text(ln, marker, boundary) + if name is not None: + dropping = name in want + if not dropping: + out.append(ln) + return "\n".join(out) diff --git a/spec/fidelity_honesty.py b/spec/fidelity_honesty.py index 7c73fb954..67056e916 100755 --- a/spec/fidelity_honesty.py +++ b/spec/fidelity_honesty.py @@ -1,4 +1,8 @@ #!/usr/bin/env python3 +# /// script +# requires-python = ">=3.13" +# dependencies = ["markdown-it-py>=4", "pyyaml>=6"] +# /// """Fidelity-honesty analysis: check the manifest's declared fidelities against fleet reality. Read-only, owner-run, not wired into CI. Reuses spec/audit.py's fleet machinery (gh, content_hash, @@ -18,7 +22,7 @@ and a repo carries anyway is hub-hosted content in a repo that should not hold it (exactly the configure.sh / settings.json bug). Each gap names its carriers, so a `retire` disposition reads as a deletion work list. -Usage: python3 spec/fidelity_honesty.py [--report] +Usage: uv run --no-project --script spec/fidelity_honesty.py [--report] """ import argparse @@ -290,7 +294,7 @@ def buckets(path): w("# Fleet divergence report") w("") w( - "Generated by `python3 spec/fidelity_honesty.py --report` - do not hand-edit. Curate dispositions in [`spec/divergences.json`][ledger] and regenerate. Each row reflects a repo's ground-truth branch at generation time. Git dates this file." + "Generated by `uv run --no-project --script spec/fidelity_honesty.py --report` - do not hand-edit. Curate dispositions in [`spec/divergences.json`][ledger] and regenerate. Each row reflects a repo's ground-truth branch at generation time. Git dates this file." ) w("") diff --git a/spec/workflow_reuse.py b/spec/workflow_reuse.py index 1b70f6435..7853e52b7 100755 --- a/spec/workflow_reuse.py +++ b/spec/workflow_reuse.py @@ -1,4 +1,8 @@ #!/usr/bin/env python3 +# /// script +# requires-python = ">=3.13" +# dependencies = ["markdown-it-py>=4", "pyyaml>=6"] +# /// """Workflow-reuse measurement: how much of the fleet's GitHub Actions YAML is a copy of a hub canonical. Read-only and owner-run. The measurement reads the live fleet, so it is not wired into CI beyond the offline @@ -21,7 +25,7 @@ The output is a fleet total (files, lines, the share of lines byte-identical to a canonical, callers), a per-workflow table, a per-repo table, and the repo-invented workflows no canonical names. -Usage: python3 spec/workflow_reuse.py [--report] [--selftest] +Usage: uv run --no-project --script spec/workflow_reuse.py [--report] [--selftest] """ import argparse @@ -201,7 +205,7 @@ def render_report(rows, unreadable, empty, hub_sha): w("# Fleet workflow reuse report") w("") w( - f'Generated by `python3 spec/workflow_reuse.py --report` at hub `{hub_sha}` - do not hand-edit. Each row reads a repo\'s ground-truth branch at generation time and compares it against the hub canonical of the same name after line-ending, action-pin, and job-needs normalization, per [`spec/fidelity-model.md`][fidelity-model] "Normalization". Git dates this file. The target model and the migration phases are in [`docs/reusable-workflows.md`][reusable-workflows].' + f'Generated by `uv run --no-project --script spec/workflow_reuse.py --report` at hub `{hub_sha}` - do not hand-edit. Each row reads a repo\'s ground-truth branch at generation time and compares it against the hub canonical of the same name after line-ending, action-pin, and job-needs normalization, per [`spec/fidelity-model.md`][fidelity-model] "Normalization". Git dates this file. The target model and the migration phases are in [`docs/reusable-workflows.md`][reusable-workflows].' ) w("") w("## Fleet Total") diff --git a/tests/requirements.txt b/tests/requirements.txt index f8d17b736..9a602841d 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,3 +1,4 @@ # Third-party libraries the scripts under test import, installed before the suite runs. # A change that adds a library to a script's inline metadata adds it here too. markdown-it-py>=4 +pyyaml>=6 diff --git a/tests/test_audit_helpers.py b/tests/test_audit_helpers.py index 659047d1f..25bfdefd6 100755 --- a/tests/test_audit_helpers.py +++ b/tests/test_audit_helpers.py @@ -532,46 +532,42 @@ def tearDownModule() -> None: ( "block scalar body dropped, key line kept", " deploy:\n name: |\n with:\n environment:\n", - " deploy:\n name: |", + "deploy:\n name:", ), ( "a dedented sibling key ends the block scalar body", " deploy:\n name: |\n with:\n with:\n environment: prod\n", - " deploy:\n name: |\n with:\n environment: prod", + "deploy:\n name:\n with:\n environment: prod", ), ( "folded scalar, strip-chomp indicator, trailing comment on the key line", " deploy:\n name: >- # a folded, strip-chomped scalar\n environment:\n", - " deploy:\n name: >- # a folded, strip-chomped scalar", + "deploy:\n name:", ), ( - # A step's `- run: |` boundary is `run:`'s column, not the dash's column. - # A sibling `env:` key genuinely at that column is real structure, not block-scalar body. "a step's `- run: |` block scalar body drops, its sibling env: mapping survives", " - run: |\n echo body\n env:\n TOKEN: xyz\n", - " - run: |\n env:\n TOKEN: xyz", + " run:\n env:\n TOKEN: xyz", ), ( - # A valid YAML anchor property (`&label`) sits between the colon and the indicator. "an anchored block scalar (`name: &label |`) is still recognized as a block-scalar key", " deploy:\n name: &lbl |\n with:\n environment:\n", - " deploy:\n name: &lbl |", + "deploy:\n name:", ), ( - # A bare sequence item (a matrix string) has no key, so its boundary is the dash's column. "a bare sequence-item block scalar (`- |`, a matrix string) still drops its body", " include:\n - |\n TOKEN\n", - " include:\n - |", + "include:", ), ( - "a folded `if: >-` keeps its body, folded onto the key line", + "a folded `if: >-` keeps its body, folded onto one line", " publish:\n if: >-\n ${{ a\n\n && b }}\n uses: x\n", - " publish:\n if: >- ${{ a && b }}\n uses: x", + "publish:\n if: ${{ a && b }}\n uses: x", ), ( "a step's folded `- if: >-` keeps its body too", " - if: >-\n ${{ a }}\n run: echo\n", - " - if: >- ${{ a }}\n run: echo", + " if: ${{ a }}\n run: echo", ), ] @@ -606,12 +602,18 @@ def test_inline_mapping(self) -> None: class CodeViewCase(unittest.TestCase): - """Holds the block-scalar handling of _code_view.""" + """Holds the block-scalar handling of code_view.""" def test_block_scalar_handling(self) -> None: for label, text, want in CODE_VIEW_CASES: with self.subTest(label): - self.assertEqual(audit._code_view(text), want) + self.assertEqual(audit.code_view(audit._compose(text)), want) + + def test_unparsed_workflow_is_one_finding(self) -> None: + got = audit.check_interface("wf", PR_CONTRACT, "jobs:\n a: [\n") + self.assertEqual( + got, [("DRIFT", "interface: wf does not parse as YAML, so its contract is unread")] + ) CANON = "line one\nline two\nline three\n" @@ -1346,7 +1348,13 @@ def test_strip_md_links(self) -> None: inner_link = "See [[docs](url)] for details." escaped_bracket = r"See [API \[docs](url) for details." msg = "description: strip_md_links behavior" - self.assertEqual(audit.strip_md_links(linked), "Utility to clean media per the spec.", msg) + refs = audit.references("x\n\n[spec-ref]: https://example.test\n") + self.assertEqual( + audit.strip_md_links(linked, refs), "Utility to clean media per the spec.", msg + ) + self.assertEqual( + audit.strip_md_links(linked), "Utility to clean media per the [spec][spec-ref].", msg + ) self.assertEqual(audit.strip_md_links("Plain intro line."), "Plain intro line.", msg) self.assertEqual(audit.strip_md_links(nested), "See API [docs] for details.", msg) self.assertEqual(audit.strip_md_links(inner_link), "See [docs] for details.", msg) @@ -3552,21 +3560,21 @@ def test_python_directories_caller(self) -> None: py_caller_value.format('&dirs "Tools\\nOther"'), {}, 1, - "an anchored escaped double-quoted value is refused", + "an anchored escaped double-quoted value is decoded", ), ( py_caller_path, py_caller_value.format('!!str "Tools\\nOther"'), {}, 1, - "a tagged escaped double-quoted value is refused", + "a tagged escaped double-quoted value is decoded", ), ( py_caller_path, py_caller_value.format('\n "Tools\\nOther"'), {}, 1, - "an escaped double-quoted value on the next line is refused", + "an escaped double-quoted value on the next line is decoded", ), ( py_caller_path, @@ -3582,7 +3590,7 @@ def test_python_directories_caller(self) -> None: py_caller_value.format('"Tools\n \\nOther"'), {"pythonDirectories": ["Tools"]}, 1, - "an escape on a double-quoted value's continuation line is refused", + "an escape on a double-quoted value's continuation line is decoded", ), ( py_caller_path, @@ -3595,22 +3603,22 @@ def test_python_directories_caller(self) -> None: py_caller_path, py_caller_value.format("&dirs >\n Tools"), {"pythonDirectories": ["Tools"]}, - 1, - "an anchored folded scalar is refused", + 0, + "an anchored folded scalar reads as its value", ), ( py_caller_path, py_caller_folded, {"pythonDirectories": ["Other", "Tools"]}, 1, - "a folded scalar is refused", + "a folded scalar joins its lines into one entry", ), ( py_caller_path, py_caller_tabbed, {"pythonDirectories": ["Too\tls"]}, - 1, - "a tabbed entry resolves to nothing", + 0, + "a tab the YAML reader refuses leaves the file to the interface check", ), ] for path_in, text_in, entry_in, expected, label in python_caller_cases: @@ -3623,10 +3631,7 @@ def test_python_directories_caller(self) -> None: escaped_got = audit.python_directories_caller_findings( py_caller_path, py_caller_escaped, {"pythonDirectories": ["Other", "Tools"]} ) - self.assertTrue( - len(escaped_got) == 1 and "backslash escape" in escaped_got[0][1], - f"an escaped double-quoted value is refused -> {escaped_got}", - ) + self.assertEqual(escaped_got, [], "an escaped double-quoted value is decoded") class ConcurrencyCallerFindingsCase(unittest.TestCase): @@ -3702,22 +3707,23 @@ def test_refusal_messages(self) -> None: ( pub_path, "concurrency: ${{ github.workflow }}-${{ github.ref }}\n -extra\njobs:\n x: {}\n", - "cannot read", - "a continuation after an inline scalar", + "group is", + "a continuation after an inline scalar folds into the group", ), ( pub_path, pub_text.replace(pub_line, " group: >-\n " + pub_value), - "cannot read", - "a folded scalar", + "", + "a folded group reads as its value", ), ( pub_path, pub_text.replace(pub_line, pub_line + "\n more"), - "cannot read", - "a continuation line", + "group is", + "a continuation line folds into the group", ), - (pub_path, "concurrency: { group: x }\n", "cannot read", "a flow mapping"), + (pub_path, "concurrency: { group: x }\n", "group is", "a flow mapping is read"), + (pub_path, "concurrency: [x]\n", "cannot read", "a sequence is refused"), ( pub_path, pub_text.replace("concurrency:\n", "concurrency: &c\n", 1), @@ -4011,8 +4017,8 @@ def test_finding_counts(self) -> None: ( pub_path, pub_text.replace(pub_line, " group: >-\n " + pub_value), - 1, - "a folded scalar is refused", + 0, + "a folded group reads as its value", ), ( pub_path, diff --git a/tests/test_build_dist.py b/tests/test_build_dist.py index 3502f87d2..a3d60427a 100755 --- a/tests/test_build_dist.py +++ b/tests/test_build_dist.py @@ -929,26 +929,26 @@ def test_fence_step_adds_the_spec_directory_to_sys_path_once(self) -> None: between calls makes the body run each time, which is what the guard is there for.""" before = len(sys.path) for _ in range(3): - build_dist._audit_module.cache_clear() + build_dist._carried_text.cache_clear() build_dist._fence_step("plain", None, 0) self.assertLessEqual(len(sys.path) - before, 1) - def test_the_audit_module_is_resolved_once_however_many_lines_are_read(self) -> None: + def test_carried_text_is_resolved_once_however_many_lines_are_read(self) -> None: """`_fence_step` reaches for it per line, so resolving it per line is what this stops.""" - build_dist._audit_module.cache_clear() + build_dist._carried_text.cache_clear() for _ in range(50): build_dist._fence_step("plain", None, 0) - self.assertEqual(build_dist._audit_module.cache_info().misses, 1) - self.assertEqual(build_dist._audit_module.cache_info().hits, 49) + self.assertEqual(build_dist._carried_text.cache_info().misses, 1) + self.assertEqual(build_dist._carried_text.cache_info().hits, 49) - def test_importing_this_module_does_not_import_audit(self) -> None: + def test_importing_this_module_does_not_import_carried_text(self) -> None: """The laziness `scripts/skills_install.py` depends on, which caching must not give up. - It imports this module on hosts whose Python predates what `audit.py` needs, and installing - never fills a region, so importing `audit` at module scope would break that path. Run in a - fresh interpreter because this test process has already imported `audit` by other means. + It imports this module on hosts whose Python predates what the spec scripts need, and installing + never fills a region, so importing `carried_text` at module scope would break that path. Run in a + fresh interpreter because this test process has already imported it by other means. The measurement is the delta across the import rather than the child's absolute state, since a `sitecustomize` or `usercustomize` hook runs before `-c` does and could import a module of @@ -957,9 +957,9 @@ def test_importing_this_module_does_not_import_audit(self) -> None: """ scripts_dir = str(Path(build_dist.__file__).parent) source = ( - "import sys; before = 'audit' in sys.modules;" + "import sys; before = 'carried_text' in sys.modules;" f" sys.path.insert(0, {scripts_dir!r}); import build_dist;" - " print(before, 'audit' in sys.modules, end='')" + " print(before, 'carried_text' in sys.modules, end='')" ) result = subprocess.run( @@ -973,9 +973,9 @@ def test_importing_this_module_does_not_import_audit(self) -> None: before, after = result.stdout.split() if before == "True": self.skipTest( - "this interpreter imports a module named audit at startup, so the import delta cannot be read here" + "this interpreter imports a module named carried_text at startup, so the import delta cannot be read here" ) - self.assertEqual(after, "False", "importing build_dist pulled in audit") + self.assertEqual(after, "False", "importing build_dist pulled in carried_text") class DeclaredDestinationCase(TreeCase): From 305a32c2f68136662ca0b519bafb34f316e2d17c Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 23 Oct 2026 14:56:27 -0700 Subject: [PATCH 03/20] Give the Audit Checks a Registry in spec/checks.json Each check audit_repo runs is an entry in spec/checks.json with an id, a family, a one-line description, the spec file it reads, its scope, and whether it runs on the hub itself. audit_repo walks the registry in order and calls check_ for each, running the per-file checks file by file so findings keep their order. The per-check hub exceptions are the hub flag. The rest stay code. The suppression ids and the driftNote id parse resolve against the registry, and spec/validate.py checks the registry's shape and that every id a driftNote names is defined. --- spec/audit.py | 1174 ++++++++++++++++++----------------- spec/checks.json | 38 ++ spec/validate.py | 54 ++ tests/test_audit_helpers.py | 34 +- tests/test_spec_validate.py | 61 ++ 5 files changed, 804 insertions(+), 557 deletions(-) create mode 100644 spec/checks.json diff --git a/spec/audit.py b/spec/audit.py index a4cbf7576..3dab49511 100755 --- a/spec/audit.py +++ b/spec/audit.py @@ -49,6 +49,7 @@ import urllib.error import urllib.parse import urllib.request +from dataclasses import dataclass, field from datetime import UTC, datetime from typing import Any @@ -82,12 +83,6 @@ # Deliberately specific, so a note recording a permanent deviation must not match. # An example of one that must not match is a note reading that there is no get-version-task and validate-task is relied on instead. PENDING_MARKERS = ["pending", "not yet", "owed", "todo", "still", "behind", "missing", "absent"] -# A driftNote may name the check that would retire it, as a parenthesized id: "(hugo.generator.pinned)". -# Deliberately unanchored. -# Both notes this form was introduced for end the sentence after the paren, so an end-anchored pattern matches neither of the two it was written to catch. -# That is the shape a matcher fails at silently: it reports nothing, and a fleet with no such note in it reports exactly the same. -CHECK_ID_RE = re.compile(r"\(([a-z][a-z0-9-]*(?:\.[a-z0-9-]+)+)\)") -DETECTION_SUPPRESSION_IDS = {"python.directories.declared": "python", "filenaming.scripts": None} def load(rel): @@ -869,12 +864,7 @@ def normalize_ruleset(payload): def check_id_owner(spec, cid): - """The project type owning a check id, and whether the catalog defines the id at all. - - Returns (None, True) for a cross-cutting check, which every repo carries and no repo declares. - The catalog is a required input rather than an optional one: resolving against an absent - catalog would report every id as undefined, which is a louder failure than reporting none. - """ + """`(owning project type or None, whether spec/checks.json or spec/project-types.json defines the id)`.""" types = spec.get("types") if not types: raise KeyError( @@ -883,24 +873,17 @@ def check_id_owner(spec, cid): for name, t in types.get("types", {}).items(): if any(c.get("id") == cid for c in t.get("checks", [])): return name, True - for dim in types.get("crossCutting", {}).values(): - if any(c.get("id") == cid for c in dim.get("checks", [])): - return None, True - return None, False + cross = (c for dim in types.get("crossCutting", {}).values() for c in dim.get("checks", [])) + known = any(c.get("id") == cid for c in cross) + return None, known or any(c["id"] == cid for c in registered_checks()) def is_suppression_id(entry, cid): - """Whether a check id a driftNote names suppresses a finding on this repo, rather than naming a check to judge. - - The note-freshness check skips such a note, and the check it suppresses reports it once stale. - DETECTION_SUPPRESSION_IDS maps each id to the type whose declaration ends the suppression, or to - None where the note suppresses on every repo. The Python id is the type model's explicit - suppression of a discovery advisory rather than a note about a forgotten type. The file-naming id - exempts the paths the note names, each a name a tool or platform fixes. - """ - if cid not in DETECTION_SUPPRESSION_IDS: + """Whether a driftNote naming `cid` suppresses a registered check's finding on this repo, rather than naming a check to judge.""" + check = next((c for c in registered_checks() if c["id"] == cid), {}) + if "suppression" not in check: return False - ending_type = DETECTION_SUPPRESSION_IDS[cid] + ending_type = check["suppression"]["endsWithType"] return ending_type is None or ending_type not in declared_types(entry) @@ -921,7 +904,7 @@ def driftnote_findings(entry, spec, open_count): out = [] for note in entry.get("driftNotes", []): quoted = quote_note(note) - for cid in CHECK_ID_RE.findall(note): + for cid in validate.CHECK_ID_RE.findall(note): if is_suppression_id(entry, cid): continue owner, known = check_id_owner(spec, cid) @@ -929,7 +912,7 @@ def driftnote_findings(entry, spec, open_count): out.append( ( "DRIFT", - f"registry: driftNote names check '{cid}', which spec/project-types.json does not define - fix the id or drop the note: {quoted}", + f"registry: driftNote names check '{cid}', which neither spec/checks.json nor spec/project-types.json defines - fix the id or drop the note: {quoted}", ) ) elif owner and owner not in declared_types(entry): @@ -2689,138 +2672,246 @@ def ground_branch_of(entry, branch=None, defaults=None): return "main" +@functools.cache +def registered_checks(): + """spec/checks.json's checks, in the order the audit runs them.""" + return tuple(load("spec/checks.json")["checks"]) + + +def check_function(check): + """The function a registered check names, `check_` with each dot and hyphen as an underscore.""" + return globals()["check_" + re.sub(r"[.-]", "_", check["id"])] + + +@dataclass +class Run: + """One repository's audit: what every registered check reads, and the findings so far.""" + + entry: dict + spec: dict + slug: str + ground: str + live: dict + main: Any + develop: Any + head: Any = None + findings: list = field(default_factory=list) + content: dict = field(default_factory=dict) + + @functools.cached_property + def hub(self): + return self.entry.get("name") == hub_repo_name() + + @functools.cached_property + def sel(self): + return repo_selectors(self.entry, self.spec["registry"].get("defaults", {})) + + @functools.cached_property + def blobs(self): + return repo_tree_blobs(self.slug, self.head) + + @functools.cached_property + def entries(self): + return blob_field(self.blobs, 0) + + @functools.cached_property + def tree(self): + return None if self.entries is None else set(self.entries) + + @functools.cached_property + def coverage_types(self): + candidates = coverage_candidate_types( + declared_types(self.entry), self.entry.get("profiles", {}) + ) + mechanisms = self.spec["secrets"].get("typeMechanisms", {}) + return coverage_claiming_types(candidates, mechanisms, self.tree) + + @functools.cached_property + def baseline(self): + """`{path: (item, sections, verbatim sections)}` for each baseline file applying here, in manifest order. + + Sections union across same-path entries, and the last fidelity-bearing entry for a path is its item. + """ + out = {} + for item in self.spec["files"]["baseline"]: + path = item["path"] + if not applies(item.get("appliesTo", "*"), self.sel): + continue + # A coverage feature file is N/A where no type claims codecov (spec/type-model.md). + if path == "codecov.yml" and not self.coverage_types: + continue + prior, needed, verbatim = out.get(path, (None, set(), set())) + needed |= set(required_sections(item, self.sel)) + verbatim |= set(verbatim_sections(item, self.sel)) + fidelity = item.get("fidelity") in ("interface", "verbatim", "intent") + out[path] = (item if fidelity else prior, needed, verbatim) + return out + + @functools.cached_property + def alternatives(self): + return declared_alternatives(self.spec["files"]["baseline"]) + + @property + def docs(self): + """README.md and HISTORY.md as read, for the checks that compare the two.""" + return { + p: self.content[p] + for p in ("README.md", "HISTORY.md") + if self.content.get(p) is not None + } + + def read_text(self, path): + url = f"repos/{self.slug}/contents/{urllib.parse.quote(path)}?ref={self.ground}" + return contents_text(gh(url)) + + def unread(self, family, path, purpose): + return ( + "DRIFT", + f"{family}: could not read {path} content on {self.ground} to {purpose} (no inline content returned); verify by hand", + ) + + def audit_repo(entry, spec, branch=None): - findings = [] # (kind, text) + """Run every registered check against one repository, returning `(findings, audited sha)`.""" slug = repo_slug(entry) - types = declared_types(entry) - model = ( - entry.get("workflowModel") - or spec["registry"].get("defaults", {}).get("workflowModel") - or "release" - ) - ground = ground_branch_of(entry, branch, spec["registry"].get("defaults", {})) - try: live = gh(f"repos/{slug}") except RuntimeError as e: return [("ERROR", str(e))], "" + run = Run( + entry, + spec, + slug, + ground_branch_of(entry, branch, spec["registry"].get("defaults", {})), + live, + gh(f"repos/{slug}/branches/main", ok404=True), + gh(f"repos/{slug}/branches/develop", ok404=True), + ) + run.head = {"main": run.main, "develop": run.develop}.get(run.ground) + if run.head is None: + run.head = gh(f"repos/{slug}/branches/{run.ground}", ok404=True) + checks = [c for c in registered_checks() if c["hub"] or not run.hub] + if run.head is None: + # Every content read would 404, so only what needs none runs. + for check in checks: + if not check.get("ground", True): + run.findings += check_function(check)(run) + missing = ( + f"branch: ground-truth branch {run.ground} does not exist, so nothing could be read" + ) + return run.findings + [("ERROR", missing)], "" + for i, check in enumerate(checks): + if check["scope"] == "repo": + run.findings += check_function(check)(run) + elif i == 0 or checks[i - 1]["scope"] == "repo": + # A run of per-file checks reads the baseline file by file, so findings group by file. + stage = list(itertools.takewhile(lambda c: c["scope"] != "repo", checks[i:])) + for path in run.baseline: + for each in stage: + if each["scope"] == "baseline" or path in run.content: + run.findings += check_function(each)(run, path) + return run.findings, run.head.get("commit", {}).get("sha", "") + + +def check_branch_facts(run): + findings = [] if run.main is not None else [("DEFECT", "branch: main does not exist")] + if bool(run.entry.get("hasDevelop")) != (run.develop is not None): + state = "exists" if run.develop is not None else "is absent" + findings.append( + ("DRIFT", f"registry: hasDevelop={run.entry.get('hasDevelop')} but develop {state}") + ) + return findings - # --- Branch facts --- - branch_main = gh(f"repos/{slug}/branches/main", ok404=True) - branch_dev = gh(f"repos/{slug}/branches/develop", ok404=True) - main_exists, dev_exists = branch_main is not None, branch_dev is not None - if not main_exists: - findings.append(("DEFECT", "branch: main does not exist")) - if bool(entry.get("hasDevelop")) != dev_exists: + +def check_branch_drift(run): + """Main-side changes develop lacks, split by direction, from the merge-base tree rather than the capped compare. + + Identical head trees are the fast path, since a merge-commit promotion leaves main permanently ahead. + """ + if run.main is None or run.develop is None: + return [] + if ( + run.main["commit"]["commit"]["tree"]["sha"] + == run.develop["commit"]["commit"]["tree"]["sha"] + ): + return [] + cmp = gh(f"repos/{run.slug}/compare/develop...main", ok404=True) + if not cmp or not cmp.get("files"): + return [] + shas = { + "base": cmp["merge_base_commit"]["commit"]["tree"]["sha"], + "develop": run.develop["commit"]["commit"]["tree"]["sha"], + "main": run.main["commit"]["commit"]["tree"]["sha"], + } + trees = {k: gh(f"repos/{run.slug}/git/trees/{sha}?recursive=1") for k, sha in shas.items()} + if not all(isinstance(t, dict) for t in trees.values()) or any( + t.get("truncated") for t in trees.values() + ): + return [ + ( + "DRIFT", + f"branch: {len(cmp['files'])}+ path(s) differ between main and develop (forward-sync or reconciliation may be needed; tree unavailable or too large to classify direction)", + ) + ] + objs = { + name: {e["path"]: e["sha"] for e in t["tree"] if e["type"] in ("blob", "commit")} + for name, t in trees.items() + } + behind, diverged = classify_branch_drift(objs["base"], objs["main"], objs["develop"]) + findings = [] + if behind: + shown = ", ".join(behind[:8]) + (" ..." if len(behind) > 8 else "") findings.append( ( "DRIFT", - f"registry: hasDevelop={entry.get('hasDevelop')} but develop {'exists' if dev_exists else 'is absent'}", + f"branch: {len(behind)} main-side path change(s) develop is behind on (forward-sync needed): {shown}", ) ) - # Resolve the branch every content read is keyed on, reusing what the branch facts already read. - # An unresolvable one is an error rather than a run, since every `?ref=` read would 404 and report the whole baseline as absent, which is a flood of letters describing the ref rather than the repo. - # The branch facts above are reported either way, since they are already read and still true. - ground_head = {"main": branch_main, "develop": branch_dev}.get(ground) - if ground_head is None: - ground_head = gh(f"repos/{slug}/branches/{ground}", ok404=True) - if ground_head is None: - return findings + [ + if diverged: + shown = ", ".join(diverged[:8]) + (" ..." if len(diverged) > 8 else "") + findings.append( ( - "ERROR", - f"branch: ground-truth branch {ground} does not exist, so nothing could be read", + "DRIFT", + f"branch: {len(diverged)} path(s) changed on both main and develop since the merge-base - reconcile before promoting (develop may already supersede main): {shown}", ) - ], "" - # Commit counts mislead here, since merge-commit promotions leave main permanently ahead while the head trees are identical, so tree equality is the no-drift fast path. - # Where the head trees differ, an empty compare files[] means develop is merely ahead, carrying no main-side changes since the merge-base, which is normal and yields no finding and no further API calls. - if ( - main_exists - and dev_exists - and branch_main["commit"]["commit"]["tree"]["sha"] - != branch_dev["commit"]["commit"]["tree"]["sha"] - ): - cmp = gh(f"repos/{slug}/compare/develop...main", ok404=True) - if cmp and cmp.get("files"): - # A non-empty files[] signals main-side changes but is not usable directly. - # It is blind to cherry-picked promotions, where develop may already hold identical content under different commit SHAs such as a promote branch, and it is capped at 300 entries. - # Derive the main-side change set from the merge-base tree instead, taking paths whose object SHA, a blob or a submodule pointer, differs from base to main, additions and deletions included and with no cap. - # Then drop paths whose objects already match at develop, since content develop already has is not content develop lacks. - # That is three recursive tree calls, so where any tree is truncated, or unexpectedly not a dict, the filter is skipped and the compare's unfiltered count is kept, which is conservative and marked. - trees = { - "base": gh( - f"repos/{slug}/git/trees/{cmp['merge_base_commit']['commit']['tree']['sha']}?recursive=1" - ), - "develop": gh( - f"repos/{slug}/git/trees/{branch_dev['commit']['commit']['tree']['sha']}?recursive=1" - ), - "main": gh( - f"repos/{slug}/git/trees/{branch_main['commit']['commit']['tree']['sha']}?recursive=1" - ), - } - if not all(isinstance(t, dict) for t in trees.values()) or any( - t.get("truncated") for t in trees.values() - ): - findings.append( - ( - "DRIFT", - f"branch: {len(cmp['files'])}+ path(s) differ between main and develop (forward-sync or reconciliation may be needed; tree unavailable or too large to classify direction)", - ) - ) - else: - objs = { - name: { - e["path"]: e["sha"] for e in t["tree"] if e["type"] in ("blob", "commit") - } - for name, t in trees.items() - } - # Split main-side drift by direction, never flagging every difference as a develop deficit. - # A path develop still holds at the merge-base is a genuine forward-sync gap. - # A path both branches moved is diverged, and develop may already supersede main. - behind, diverged = classify_branch_drift( - objs["base"], objs["main"], objs["develop"] - ) - if behind: - shown = ", ".join(behind[:8]) + (" ..." if len(behind) > 8 else "") - findings.append( - ( - "DRIFT", - f"branch: {len(behind)} main-side path change(s) develop is behind on (forward-sync needed): {shown}", - ) - ) - if diverged: - shown = ", ".join(diverged[:8]) + (" ..." if len(diverged) > 8 else "") - findings.append( - ( - "DRIFT", - f"branch: {len(diverged)} path(s) changed on both main and develop since the merge-base - reconcile before promoting (develop may already supersede main): {shown}", - ) - ) + ) + return findings - # --- General settings --- - expected = dict(spec["settings"]) - expected["has_discussions"] = not live.get("private", True) - for key in SETTINGS_KEYS + ["has_discussions"]: - if key in expected and live.get(key) != expected[key]: - findings.append( - ("DEFECT", f"settings: {key} live={live.get(key)} expected={expected[key]}") - ) - if main_exists and live.get("default_branch") != "main": + +def check_settings_general(run): + expected = dict(run.spec["settings"]) + expected["has_discussions"] = not run.live.get("private", True) + findings = [ + ("DEFECT", f"settings: {key} live={run.live.get(key)} expected={expected[key]}") + for key in SETTINGS_KEYS + ["has_discussions"] + if key in expected and run.live.get(key) != expected[key] + ] + if run.main is not None and run.live.get("default_branch") != "main": findings.append( - ("DEFECT", f"settings: default_branch is {live.get('default_branch')}, expected main") + ( + "DEFECT", + f"settings: default_branch is {run.live.get('default_branch')}, expected main", + ) ) + return findings + - # --- Rulesets --- +def check_rulesets_payload(run): + model = ( + run.entry.get("workflowModel") + or run.spec["registry"].get("defaults", {}).get("workflowModel") + or "release" + ) dev_payload = ( "repo-config/operational/develop.json" if model == "operational" else "repo-config/develop.json" ) - expect_rulesets = {"develop": load(dev_payload), "main": load("repo-config/main.json")} - # No ok404: a repo with no rulesets returns an empty list, so a 404 means the call failed - # (access/rename) and must surface as ERROR, not read as "no rulesets". - live_list = gh(f"repos/{slug}/rulesets?per_page=100") or [] - live_names = [r["name"] for r in live_list] - for name, payload in expect_rulesets.items(): + expect = {"develop": load(dev_payload), "main": load("repo-config/main.json")} + # A repo with no rulesets returns an empty list, so a 404 is a failed call and raises. + live_list = gh(f"repos/{run.slug}/rulesets?per_page=100") or [] + findings = [] + for name, payload in expect.items(): ids = [r["id"] for r in live_list if r["name"] == name] if not ids: findings.append(("DEFECT", f"ruleset: {name} missing")) @@ -2829,80 +2920,74 @@ def audit_repo(entry, spec, branch=None): findings.append( ("DRIFT", f"ruleset: {len(ids)} rulesets named {name} (resolve the duplicates)") ) - live_rs = gh(f"repos/{slug}/rulesets/{ids[0]}") + live_rs = gh(f"repos/{run.slug}/rulesets/{ids[0]}") if normalize_ruleset(live_rs) != normalize_ruleset(payload): - findings.append( - ( - "DEFECT", - f"ruleset: {name} diverges from {dev_payload if name == 'develop' else 'repo-config/main.json'} (normalized diff)", - ) - ) - for stray in [n for n in live_names if n not in expect_rulesets]: + source = dev_payload if name == "develop" else "repo-config/main.json" + findings.append(("DEFECT", f"ruleset: {name} diverges from {source} (normalized diff)")) + for stray in [r["name"] for r in live_list if r["name"] not in expect]: findings.append(("DRIFT", f"ruleset: stray ruleset '{stray}'")) + return findings - # --- Secrets (names only) --- - secrets = spec["secrets"] - # The codecov coverage requirement, meaning the CODECOV_TOKEN secret and the codecov.yml file, is claimed by a type only where the repo carries tests for that type. - # A language with no test suite has none, since the validator's leg never runs and the token and the codecov.yml would gate on a report the pipeline cannot produce. - # The tree is read here rather than at the verbatim-tree section below so one call answers both, and the finding for an unreadable tree stays where it was. - carried_blobs = repo_tree_blobs(slug, ground_head) - carried_entries = blob_field(carried_blobs, 0) - tree = None if carried_entries is None else set(carried_entries) - coverage_types = coverage_claiming_types( - coverage_candidate_types(types, entry.get("profiles", {})), - secrets.get("typeMechanisms", {}), - tree, - ) - coverage_active = bool(coverage_types) - # --- Python presence and directory coverage --- - # A tracked .py file is a fact about the tree, read once here and checked against the registry's own - # declarations rather than against what any one caller's workflow happens to gate today. - findings.extend(python_type_findings(entry, tree)) - findings.extend(python_directory_coverage_findings(entry, tree)) - findings.extend(python_undeclared_root_findings(entry, tree)) - - def read_text(path): - return contents_text(gh(f"repos/{slug}/contents/{urllib.parse.quote(path)}?ref={ground}")) - - findings.extend(file_naming_findings(entry, blob_field(carried_blobs, 1), read_text)) - findings.extend(secret_stem_findings(tree)) + +def check_python_directories_declared(run): + return python_type_findings(run.entry, run.tree) + + +def check_python_directories_coverage(run): + return python_directory_coverage_findings(run.entry, run.tree) + + +def check_python_directories_root(run): + return python_undeclared_root_findings(run.entry, run.tree) + + +def check_filenaming_scripts(run): + return file_naming_findings(run.entry, blob_field(run.blobs, 1), run.read_text) + + +def check_secrets_stems(run): + return secret_stem_findings(run.tree) + + +def check_secrets_names(run): + """Secret names per store, never values, against what the applicable mechanisms require and forbid. + + The codecov mechanism follows the coverage-claiming types, so a language with no tests requires no token. + """ + secrets = run.spec["secrets"] stores = {} - # There is no ok404 here, since an empty store returns {"secrets": []}, so a 404 or 403 from permissions or a rename must surface as ERROR rather than cascade into false missing-secret DEFECTs. - for store, path in [ - ("actions", f"repos/{slug}/actions/secrets?per_page=100"), - ("dependabot", f"repos/{slug}/dependabot/secrets?per_page=100"), - ]: - data = gh(path) + # No ok404, since an empty store returns an empty list and a 404 is a failed call. + for store in ("actions", "dependabot"): + data = gh(f"repos/{run.slug}/{store}/secrets?per_page=100") stores[store] = {s["name"] for s in (data or {}).get("secrets", [])} mechanisms = [ - secrets["targetMechanisms"].get(p.get("target")) for p in entry.get("publish", []) + secrets["targetMechanisms"].get(p.get("target")) for p in run.entry.get("publish", []) ] - # The codecov mechanism follows coverage_types rather than the profile, so a language with no tests requires no CODECOV_TOKEN. - for name in types: + for name in declared_types(run.entry): mechanism = secrets.get("typeMechanisms", {}).get(name) - if mechanism == "codecov" and name not in coverage_types: - continue - mechanisms.append(mechanism) + if mechanism != "codecov" or name in run.coverage_types: + mechanisms.append(mechanism) claimed = [secrets["mechanisms"][m] for m in mechanisms if m and m in secrets["mechanisms"]] - required_by_store = {"actions": set(), "dependabot": set()} + required = {"actions": set(), "dependabot": set()} for store in secrets["baseline"].get("stores", []): - required_by_store[store] |= set(secrets["baseline"].get("requires", [])) + required[store] |= set(secrets["baseline"].get("requires", [])) for mech in claimed: for store in mech.get("stores", []): - required_by_store[store] |= set(mech.get("requires", [])) - # The registry requiredSecrets entries are the domain-specific additions, per STANDUP.md, being requiredSecrets plus the implicit baseline. - # Mechanism-mapped names already carry their stores above, and unmapped ones are expected in the actions store and count as claimed, never stale. - required_by_store["actions"] |= set(entry.get("requiredSecrets", [])) + required[store] |= set(mech.get("requires", [])) + # The registry requiredSecrets are the domain-specific additions, per STANDUP.md, expected in the actions store. + required["actions"] |= set(run.entry.get("requiredSecrets", [])) forbidden = set(secrets["baseline"].get("forbids", [])) for mech in claimed: forbidden |= set(mech.get("forbids", [])) - for store, required in required_by_store.items(): - for name in sorted(required - stores[store]): - findings.append(("DEFECT", f"secrets: {name} missing from the {store} store")) + findings = [ + ("DEFECT", f"secrets: {name} missing from the {store} store") + for store, names in required.items() + for name in sorted(names - stores[store]) + ] + claimed_names = required["actions"] | required["dependabot"] for store, present in stores.items(): for name in sorted(present & forbidden): findings.append(("DEFECT", f"secrets: forbidden {name} present in the {store} store")) - claimed_names = required_by_store["actions"] | required_by_store["dependabot"] for name in sorted(present - claimed_names): findings.append( ( @@ -2910,397 +2995,374 @@ def read_text(path): f"secrets: {name} in the {store} store is claimed by no applicable mechanism (stale?)", ) ) + return findings - # --- Dependabot ecosystem coverage --- - # A repo's tree implies Dependabot ecosystems it must track, being github-actions where it ships workflows and devcontainers where it ships a .devcontainer. - # Without the first, the action versions those workflows reference go stale and a merge-bot then has no PRs to auto-merge. - # This asserts an implied ecosystem's *presence* only. - # Whether each declared ecosystem dual-targets main and develop, which is the fleet norm, is verified by inspection rather than here. - # It only runs where dependabot.yml exists, since its absence is already a file-presence LETTER below. - # Language ecosystems such as nuget, uv and npm are directory-scoped and not yet cross-checked here. - db = gh(f"repos/{slug}/contents/.github/dependabot.yml?ref={ground}", ok404=True) - db_text = contents_text(db) - if db_text: - declared = dependabot_ecosystems(db_text) - implied = {} - workflows = gh(f"repos/{slug}/contents/.github/workflows?ref={ground}", ok404=True) - if isinstance(workflows, list) and any( - e["name"].endswith((".yml", ".yaml")) for e in workflows - ): - implied["github-actions"] = ".github/workflows/ is present" - if gh(f"repos/{slug}/contents/.devcontainer?ref={ground}", ok404=True) is not None: - implied["devcontainers"] = ".devcontainer/ is present" - for eco, why in sorted(implied.items()): - if eco not in declared: - findings.append( - ( - "DRIFT", - f"dependabot: {eco} ecosystem not declared though {why}; add it for both main and develop per the fleet norm", - ) - ) - # --- File and section presence on the ground-truth branch --- - # The appliesTo selector is matched against the repo's full selector set, being types, workflowModel, releaseTrigger and consumerModel, so the release and operational develop rulesets are two data entries rather than a code swap. - # Required sections union across same-path entries. - # A carried Markdown file must contain each heading scoped to this repo. - # A rename reads as missing and equivalence is judged by hand, so a missing section is DRIFT, a hint to verify, and never a LETTER. - sel = repo_selectors(entry, spec["registry"].get("defaults", {})) - wanted_sections = {} # path -> set of required section names, unioned across applicable entries - verbatim_secs = {} # path -> set of section names checked byte-for-byte against the hub canonical - doc_texts = {} # README.md / HISTORY.md content, retained for the mirror check - check_item = {} # path -> entry, for a fidelity interface/verbatim entry (last applicable wins per path) - alternatives = declared_alternatives(spec["files"]["baseline"]) - path_order = [] - for item in spec["files"]["baseline"]: - if not applies(item.get("appliesTo", "*"), sel): - continue - path = item["path"] - if path == "codecov.yml" and not coverage_active: - continue # coverage feature file: N/A when no type claims codecov at build profile with tests present (spec/type-model.md) - if path not in wanted_sections: - wanted_sections[path] = set() - verbatim_secs[path] = set() - path_order.append(path) - wanted_sections[path].update(required_sections(item, sel)) - verbatim_secs[path].update(verbatim_sections(item, sel)) - if item.get("fidelity") in ("interface", "verbatim", "intent"): - check_item[path] = item - for path in path_order: - item = check_item.get(path) - fid = item.get("fidelity") if item else "presence" - content, text, file_findings = baseline_file_findings( - slug, - ground, +def check_dependabot_ecosystems(run): + """An ecosystem the tree implies, github-actions for workflows and devcontainers for a .devcontainer, is declared. + + Presence only. An absent dependabot.yml is already a file LETTER. + """ + text = contents_text( + gh(f"repos/{run.slug}/contents/.github/dependabot.yml?ref={run.ground}", ok404=True) + ) + if not text: + return [] + declared = dependabot_ecosystems(text) + implied = {} + workflows = gh(f"repos/{run.slug}/contents/.github/workflows?ref={run.ground}", ok404=True) + if isinstance(workflows, list) and any( + e["name"].endswith((".yml", ".yaml")) for e in workflows + ): + implied["github-actions"] = ".github/workflows/ is present" + if gh(f"repos/{run.slug}/contents/.devcontainer?ref={run.ground}", ok404=True) is not None: + implied["devcontainers"] = ".devcontainer/ is present" + return [ + ( + "DRIFT", + f"dependabot: {eco} ecosystem not declared though {why}; add it for both main and develop per the fleet norm", + ) + for eco, why in sorted(implied.items()) + if eco not in declared + ] + + +def check_files_presence(run, path): + content, text, findings = baseline_file_findings( + run.slug, run.ground, path, run.alternatives.get(path, []), run.baseline[path][0], run.hub + ) + if content is not None: + run.content[path] = text + return findings + + +def _fidelity(run, path): + item = run.baseline[path][0] + return item, (item or {}).get("fidelity", "presence") + + +def check_interface_contract(run, path): + item, fid = _fidelity(run, path) + if fid != "interface": + return [] + if run.content[path] is None: + return [run.unread("interface", path, "verify its contract")] + return check_interface(path, item.get("contract", {}), run.content[path]) + + +def check_interface_python_directories(run, path): + if _fidelity(run, path)[1] != "interface" or run.content[path] is None: + return [] + return python_directories_caller_findings(path, run.content[path], run.entry) + + +def check_interface_concurrency(run, path): + if _fidelity(run, path)[1] != "interface" or run.content[path] is None: + return [] + return concurrency_caller_findings(path, run.content[path]) + + +def check_verbatim_jobs(run, path): + item, fid = _fidelity(run, path) + if fid != "interface" or run.content[path] is None: + return [] + return [ + finding + for job in item.get("contract", {}).get("verbatimJobs", []) + for finding in check_verbatim( + f"{path} job '{job}'", + run.content[path], + item.get("reference") or path, + extract=lambda t, j=job: split_jobs(t).get(j), + ) + ] + + +def check_verbatim_file(run, path): + item, fid = _fidelity(run, path) + if fid != "verbatim": + return [] + if run.content[path] is None: + return [run.unread("verbatim", path, "compare")] + return check_verbatim(path, run.content[path], item.get("reference") or path) + + +def _sections(run, path): + """The sections a Markdown file owes here, `(presence, verbatim)`, both empty for any other file.""" + _, needed, verbatim = run.baseline[path] + return (needed, verbatim) if path.endswith(".md") else (set(), set()) + + +def check_sections_presence(run, path): + needed, verbatim = _sections(run, path) + if not (needed or verbatim): + return [] + if run.content[path] is None: + return [run.unread("section", path, "verify sections")] + present = heading_texts(run.content[path]) + return [ + ( + "DRIFT", + f"section: '{name}' not found as a heading in {path} on {run.ground} (renamed or missing; verify intent per AUDIT.md section 7)", + ) + for name in sorted(needed) + if name.strip().lower() not in present + ] + + +def check_sections_verbatim(run, path): + needed, verbatim = _sections(run, path) + if not (needed or verbatim) or run.content[path] is None: + return [] + return [ + finding + for name in sorted(verbatim) + for finding in check_verbatim( + f"{path} section '{name}'", + run.content[path], path, - alternatives.get(path, []), - item, - entry.get("name") == hub_repo_name(), + extract=lambda t, n=name: extract_section(t, n), ) - findings.extend(file_findings) - if content is None: - continue - if path in ("README.md", "HISTORY.md") and text is not None: - doc_texts[path] = text # retained for the README/HISTORY mirror check below - # Interface conformance (name + wiring) plus any verbatim job regions the contract pins. - if item is not None and fid == "interface": - if text is None: - findings.append( - ( - "DRIFT", - f"interface: could not read {path} content on {ground} to verify its contract (no inline content returned); verify by hand", - ) - ) - else: - contract = item.get("contract", {}) - findings.extend(check_interface(path, contract, text)) - findings.extend(python_directories_caller_findings(path, text, entry)) - findings.extend(concurrency_caller_findings(path, text)) - canonical_rel = item.get("reference") or path - for job in contract.get("verbatimJobs", []): - findings.extend( - check_verbatim( - f"{path} job '{job}'", - text, - canonical_rel, - extract=lambda t, j=job: split_jobs(t).get(j), - ) - ) - # Whole-file verbatim: byte-identical to the hub's canonical after EOL normalization. - elif item is not None and fid == "verbatim": - if text is None: - findings.append( - ( - "DRIFT", - f"verbatim: could not read {path} content on {ground} to compare (no inline content returned); verify by hand", - ) - ) - else: - findings.extend(check_verbatim(path, text, item.get("reference") or path)) - # Heading-based presence is only meaningful for Markdown. - # A section named on a non-md file (a tasks.json task group) is not checked, only the file's presence, and RESYNC.md aligns it by hand. - needed = wanted_sections[path] - verbatim_needed = verbatim_secs[path] - if (needed or verbatim_needed) and path.endswith(".md"): - if text is None: - # Fail loud rather than skip silently, since the contents API returned no inline content, which happens for an oversized file, a symlink or a submodule. - # The section check could not run, so surface that rather than a false clean. - findings.append( - ( - "DRIFT", - f"section: could not read {path} content on {ground} to verify sections (no inline content returned); verify by hand", - ) - ) - else: - present = heading_texts(text) - for name in sorted(needed): - if name.strip().lower() not in present: - findings.append( - ( - "DRIFT", - f"section: '{name}' not found as a heading in {path} on {ground} (renamed or missing; verify intent per AUDIT.md section 7)", - ) - ) - # A verbatim section must match the hub's canonical byte-for-byte once EOL-normalized, like a verbatim file but scoped to the one `## ` region. - # A universal rule block then cannot drift or fall behind a newly added rule while its heading still passes the presence check. - for name in sorted(verbatim_needed): - findings.extend( - check_verbatim( - f"{path} section '{name}'", - text, - path, - extract=lambda t, n=name: extract_section(t, n), - ) - ) - # The undeclared-section advisory, per spec/section-model.md, treats an H2 the manifest does not declare as a candidate duplicate of a verbatim section, or as repo-specific content to relocate. - # It is advisory only, since the reconciliation can end in promoting the rule here or relocating it, so it points at that choice and never fails. - # It covers UNDECLARED_HEADING_SCANNED, not only AGENTS.md and GOVERNANCE.md, and never names which destination file an undeclared heading belongs in. - # Skip the hub itself, since its copies are the source and legitimately hold hub-only sections, Repository Onboarding and Conformance being one, that are deliberately not carried. - # A downstream repo carrying such a section is still flagged, which is the point. - if path in UNDECLARED_HEADING_SCANNED and entry.get("name") != hub_repo_name(): - declared = {n.strip().lower() for n in (needed | verbatim_needed)} - for h in undeclared_h2_headings(text, declared): - findings.append( - ( - "DRIFT", - f"section: '{h}' in {path} is not a declared section - reconcile it (a duplicate of a verbatim section, or repo-specific content that moves to a topical doc), or confirm it is intentional (spec/section-model.md)", - ) - ) + ] - # --- Carried files must not reference the template repo --- - # The coordination flow is machinery a consumer should not see, so a carried file states the behavior rather than the destination. - # A stale "report drift upstream" paragraph once spread this way. - # Skip the hub itself, whose own carried files are the source, where naming the repo they live in is correct. - # A downstream repo naming it in prose it owns is still flagged, which is the point. - # Verbatim sections are excised first, and template_ref_outside_verbatim carries why that is not a loophole. - # Sited here, in the file loop, so the scan reuses the content already fetched for the section checks and reads the same selector-resolved verbatim list they were judged against. - if path in TEMPLATE_REF_SCANNED and entry.get("name") != hub_repo_name(): - if text is None: - findings.append( - ( - "DRIFT", - f"carried: could not read {path} content on {ground} to scan for a coordination reference (no inline content returned); verify by hand", - ) - ) - elif template_ref_outside_verbatim(text, verbatim_secs[path], hub_repo_name()): - findings.append( - ( - "DRIFT", - f"carried: {path} references the template repo by name or link outside its verbatim sections (the coordination flow is machinery this repo's readers should not see; state the behavior, not the destination)", - ) - ) - # --- Instruction documents carry no version literal or commit SHA --- - # The hub is scanned whole, verbatim sections included, since those are the copies every repo carries and no other run can see a literal inside them. - if path in VERSION_LITERAL_SCANNED: - if text is None: +def check_sections_undeclared(run, path): + """Advisory: an undeclared H2 is a candidate duplicate of a verbatim section, or content to relocate.""" + needed, verbatim = _sections(run, path) + if not (needed or verbatim) or run.content[path] is None: + return [] + if path not in UNDECLARED_HEADING_SCANNED: + return [] + return [ + ( + "DRIFT", + f"section: '{h}' in {path} is not a declared section - reconcile it (a duplicate of a verbatim section, or repo-specific content that moves to a topical doc), or confirm it is intentional (spec/section-model.md)", + ) + for h in undeclared_h2_headings(run.content[path], needed | verbatim) + ] + + +def check_carried_template_ref(run, path): + if path not in TEMPLATE_REF_SCANNED: + return [] + if run.content[path] is None: + return [run.unread("carried", path, "scan for a coordination reference")] + if not template_ref_outside_verbatim(run.content[path], run.baseline[path][2], hub_repo_name()): + return [] + return [ + ( + "DRIFT", + f"carried: {path} references the template repo by name or link outside its verbatim sections (the coordination flow is machinery this repo's readers should not see; state the behavior, not the destination)", + ) + ] + + +def check_carried_version_literal(run, path): + if path not in VERSION_LITERAL_SCANNED: + return [] + if run.content[path] is None: + return [run.unread("carried", path, "scan for a version literal")] + # The hub is scanned whole, since its verbatim sections are the copies every repo carries. + excised = set() if run.hub else run.baseline[path][2] + literals = version_literals_outside_verbatim(run.content[path], excised) + if not literals: + return [] + return [ + ( + "DRIFT", + f"carried: {path} names a three-part version or a commit SHA, full or abbreviated ({', '.join(literals)}); name a pin by its mechanism, never its value, and write a versioning example with a placeholder such as 1.0.N (GOVERNANCE.md, Documentation Style Conventions, References)", + ) + ] + + +def check_tree_readable(run): + if run.entries is not None: + return [] + return [ + ( + "DRIFT", + f"tree: could not read the file tree for {run.slug}@{run.ground} in full, so verbatim trees, script file names, and secret-file stems are undecided", + ) + ] + + +def check_tree_verbatim(run): + if run.entries is None: + return [] + findings = [] + for declaration in run.spec["files"].get("trees", []): + if not applies(declaration.get("appliesTo", "*"), run.sel): + continue + source_root = declaration["source"].rstrip("/") + target_root = declaration["target"].rstrip("/") + source_files = { + path.removeprefix(source_root + "/"): path + for path in hub_tracked() + if path.startswith(source_root + "/") + and tree_path_included(path.removeprefix(source_root + "/"), declaration["include"]) + } + target_files = { + path.removeprefix(target_root + "/"): path + for path in run.entries + if path.startswith(target_root + "/") + } + for relative in sorted(set(source_files) - set(target_files)): + findings.append(("LETTER", f"tree: {target_root}/{relative} is absent on {run.ground}")) + for relative in sorted(set(source_files) & set(target_files)): + findings.extend( + _tree_file_findings(source_files[relative], target_files[relative], run.entries) + ) + if declaration.get("prune"): + for relative in sorted(set(target_files) - set(source_files)): findings.append( - ( - "DRIFT", - f"carried: could not read {path} content on {ground} to scan for a version literal (no inline content returned); verify by hand", - ) + ("DRIFT", f"tree: {target_root}/{relative} is extra under a pruned target") ) - else: - excised = set() if entry.get("name") == hub_repo_name() else verbatim_secs[path] - literals = version_literals_outside_verbatim(text, excised) - if literals: - findings.append( - ( - "DRIFT", - f"carried: {path} names a three-part version or a commit SHA, full or abbreviated ({', '.join(literals)}); name a pin by its mechanism, never its value, and write a versioning example with a placeholder such as 1.0.N (GOVERNANCE.md, Documentation Style Conventions, References)", - ) - ) + return findings - # --- Manifest-owned verbatim trees --- - # carried_entries was read once above, where the coverage applicability check needed it first. - if carried_entries is None: - findings.append( + +def _tree_file_findings(source_path, target_path, entries): + try: + source_sha = canonical_blob_sha(source_path) + except OSError as exc: + return [ + ("DRIFT", f"tree: canonical {source_path} is unreadable, comparison undecided: {exc}") + ] + target_sha = entries[target_path] + if source_sha == target_sha: + return [] + if not target_sha: + return [ ( "DRIFT", - f"tree: could not read the file tree for {slug}@{ground} in full, so verbatim trees, script file names, and secret-file stems are undecided", + f"tree: {target_path} differs but its blob identity is unreadable, comparison undecided", ) + ] + if git_blob_in_file_history(source_path, target_sha): + return [ + ("DRIFT", f"tree: {target_path} matches a past hub revision, not the current canonical") + ] + return [ + ( + "DRIFT", + f"tree: {target_path} differs from the canonical and matches no past hub revision", ) - elif entry.get("name") != hub_repo_name(): - for declaration in spec["files"].get("trees", []): - if not applies(declaration.get("appliesTo", "*"), sel): - continue - source_root = declaration["source"].rstrip("/") - target_root = declaration["target"].rstrip("/") - source_files = { - path.removeprefix(source_root + "/"): path - for path in hub_tracked() - if path.startswith(source_root + "/") - and tree_path_included(path.removeprefix(source_root + "/"), declaration["include"]) - } - target_files = { - path.removeprefix(target_root + "/"): path - for path in carried_entries - if path.startswith(target_root + "/") - } - for relative in sorted(set(source_files) - set(target_files)): - findings.append(("LETTER", f"tree: {target_root}/{relative} is absent on {ground}")) - for relative in sorted(set(source_files) & set(target_files)): - source_path = source_files[relative] - try: - source_sha = canonical_blob_sha(source_path) - except OSError as exc: - findings.append( - ( - "DRIFT", - f"tree: canonical {source_path} is unreadable, comparison undecided: {exc}", - ) - ) - continue - target_path = target_files[relative] - target_sha = carried_entries[target_path] - if source_sha == target_sha: - continue - if not target_sha: - findings.append( - ( - "DRIFT", - f"tree: {target_path} differs but its blob identity is unreadable, comparison undecided", - ) - ) - continue - verdict = ( - "stale" if git_blob_in_file_history(source_path, target_sha) else "modified" - ) - if verdict == "stale": - findings.append( - ( - "DRIFT", - f"tree: {target_path} matches a past hub revision, not the current canonical", - ) - ) - else: - findings.append( - ( - "DRIFT", - f"tree: {target_path} differs from the canonical and matches no past hub revision", - ) - ) - if declaration.get("prune"): - for relative in sorted(set(target_files) - set(source_files)): - findings.append( - ("DRIFT", f"tree: {target_root}/{relative} is extra under a pruned target") - ) + ] - # --- Hub-only files a repo carries and should not --- - # The manifest declares what a repo carries, so a path the hub tracks and the manifest omits is hub-hosted content per GOVERNANCE.md "Hub-Hosted Tooling". - # A downstream copy of one is drift whose remedy is a deletion rather than a re-vendor, which no other check reports: every check above reads a path the manifest names, so nothing looks at what a repo carries beyond the baseline. - # Skipped for the hub, whose own tracked files are the source and are all "hub-only" by construction. - if entry.get("name") != hub_repo_name(): - carried = None if carried_entries is None else set(carried_entries) - if carried is None: - findings.append( - ( - "DRIFT", - f"hub-only: could not read the file tree for {slug}@{ground} in full, so no hub-hosted file this repo carries was checked; verify by hand", - ) + +def check_hubonly_carried(run): + """A path the hub tracks and the manifest omits is hub-hosted, so a downstream copy converges by deletion.""" + if run.tree is None: + return [ + ( + "DRIFT", + f"hub-only: could not read the file tree for {run.slug}@{run.ground} in full, so no hub-hosted file this repo carries was checked; verify by hand", ) + ] + disp = gap_dispositions(run.spec) + findings = [] + for path in sorted(run.tree & hub_only_paths(run.spec)): + kind = disp.get(path, ("", ""))[0] + if kind == "accepted": + continue + if kind == "retire": + text = f"hub-only: delete this repo's {path} - the hub hosts it and no repo carries it (spec/divergences.json 'gaps' records why, and what to reach instead)" + elif kind: + text = f"hub-only: {path} is undeclared in spec/files.json and this repo carries it - the ledger dispositions it '{kind}', so settle that before converging" else: - disp = gap_dispositions(spec) - for path in sorted(carried & hub_only_paths(spec)): - kind = disp.get(path, ("", ""))[0] - # The ledger's reason is not quoted here, since slicing a sentence out of it cuts at the first period, which lands inside a filename like GOVERNANCE.md. - # The path names the row to read instead. - if kind == "accepted": - continue # a triaged permanent divergence, so not a finding - if kind == "retire": - findings.append( - ( - "DRIFT", - f"hub-only: delete this repo's {path} - the hub hosts it and no repo carries it (spec/divergences.json 'gaps' records why, and what to reach instead)", - ) - ) - elif kind: - findings.append( - ( - "DRIFT", - f"hub-only: {path} is undeclared in spec/files.json and this repo carries it - the ledger dispositions it '{kind}', so settle that before converging", - ) - ) - else: - findings.append( - ( - "DRIFT", - f"hub-only: {path} is undeclared in spec/files.json and this repo carries it - read the file and triage it in spec/divergences.json 'gaps' (this repo's own content at a shared path, a carry to declare, or a hub copy to delete)", - ) - ) + text = f"hub-only: {path} is undeclared in spec/files.json and this repo carries it - read the file and triage it in spec/divergences.json 'gaps' (this repo's own content at a shared path, a carry to declare, or a hub copy to delete)" + findings.append(("DRIFT", text)) + return findings + - # --- HISTORY.md mirrors the README opening --- - # Per spec/readme-structure.md "HISTORY.md", the changelog opens as the README's twin, carrying the same H1 title and the same tagline. - # The mirror is the tagline alone, not the whole intro region: a README may carry further clarifying paragraphs, and HISTORY.md does not repeat them. - # It is checked only where both files were readable, since absence is already a file LETTER above. - if "README.md" in doc_texts and "HISTORY.md" in doc_texts: - r_title, r_intro = title_and_intro(doc_texts["README.md"]) - h_title, h_intro = title_and_intro(doc_texts["HISTORY.md"]) - if r_title != h_title: +def check_history_mirror(run): + """HISTORY.md opens as the README's twin, the same H1 and the same tagline (spec/readme-structure.md).""" + if len(run.docs) < 2: + return [] + r_title, r_intro = title_and_intro(run.docs["README.md"]) + h_title, h_intro = title_and_intro(run.docs["HISTORY.md"]) + if r_title != h_title: + return [ + ( + "LETTER", + f"history: HISTORY.md title '{h_title}' does not match README.md title '{r_title}' - the changelog opens as the README's twin (spec/readme-structure.md)", + ) + ] + if tagline(r_intro) != tagline(h_intro): + return [ + ( + "LETTER", + "history: HISTORY.md tagline does not mirror the README tagline - copy the README's first line after the H1 (spec/readme-structure.md)", + ) + ] + return [] + + +def check_readme_description(run): + return description_findings(run.docs, run.entry, run.live, run.slug) + + +def _readme(run): + """The README text where the README model is loaded and the README was read, else None.""" + return run.docs.get("README.md") if run.spec.get("readme") else None + + +def check_readme_sections(run): + text = _readme(run) + if text is None: + return [] + return readme_section_findings( + text, run.spec["readme"], run.sel, not run.live.get("private", False) + ) + + +def check_readme_shields(run): + text = _readme(run) + return [] if text is None else readme_shield_findings(text, run.spec["readme"], run.entry) + + +def check_readme_links(run): + text = _readme(run) + return [] if text is None else readme_link_findings(text, run.spec["readme"], run.slug) + + +def check_readme_tools(run): + text = _readme(run) + if text is None or not run.spec.get("tools"): + return [] + return third_party_tool_findings(text, run.spec["tools"]) + + +def check_cspell_workspace(run): + """cspell.json is the one word list, so a workspace file's own cSpell list duplicates it.""" + if gh(f"repos/{run.slug}/contents/cspell.json?ref={run.ground}", ok404=True) is None: + return [] + findings = [] + for it in gh(f"repos/{run.slug}/contents/?ref={run.ground}", ok404=True) or []: + ws_name = it.get("name", "") if isinstance(it, dict) else "" + if not ws_name.endswith(".code-workspace"): + continue + ws_text = contents_text( + gh(f"repos/{run.slug}/contents/{ws_name}?ref={run.ground}", ok404=True) + ) + if ws_text is None: findings.append( ( - "LETTER", - f"history: HISTORY.md title '{h_title}' does not match README.md title '{r_title}' - the changelog opens as the README's twin (spec/readme-structure.md)", + "DRIFT", + f"cspell: could not read {ws_name} on {run.ground} to check for a duplicated word list; verify by hand", ) ) - elif tagline(r_intro) != tagline(h_intro): + elif workspace_cspell_words(ws_text): findings.append( ( "LETTER", - "history: HISTORY.md tagline does not mirror the README tagline - copy the README's first line after the H1 (spec/readme-structure.md)", + f"cspell: {ws_name} carries a cSpell word list while cspell.json is the single source of truth - delete the workspace copy (CODESTYLE.md Markdown and Spelling)", ) ) + return findings - # --- README title and intro are the one canonical short description --- - # See description_findings() for the mirror set (README, GitHub About, Docker Hub) and the declared-field precedence. - findings += description_findings(doc_texts, entry, live, slug) - - # --- README section order, shield classes, and license-shield placement --- - # The readme-structure dimension, driven by the declared model in spec/readme-sections.json rather than by prose. - # Sited after the title and intro checks so one README read serves both, and guarded on the model being loaded, since the selftest builds a spec without it. - readme_model = spec.get("readme") - if readme_model and "README.md" in doc_texts: - readme_text = doc_texts["README.md"] - findings += readme_section_findings( - readme_text, readme_model, sel, not live.get("private", False) - ) - findings += readme_shield_findings(readme_text, readme_model, entry) - findings += readme_link_findings(readme_text, readme_model, slug) - if spec.get("tools"): - findings += third_party_tool_findings(readme_text, spec["tools"]) - - # --- cspell single source of truth --- - # Per CODESTYLE.md "Markdown and Spelling", cspell.json is the one word list, and a cSpell words block left in a *.code-workspace duplicates it and silently drifts. - # It is checked only where cspell.json is carried, since its absence is already a file LETTER above, and a workspace list with no cspell.json is that same finding. - if gh(f"repos/{slug}/contents/cspell.json?ref={ground}", ok404=True) is not None: - root_entries = gh(f"repos/{slug}/contents/?ref={ground}", ok404=True) or [] - for it in root_entries: - ws_name = it.get("name", "") if isinstance(it, dict) else "" - if not ws_name.endswith(".code-workspace"): - continue - ws = gh(f"repos/{slug}/contents/{ws_name}?ref={ground}", ok404=True) - ws_text = contents_text(ws) - if ws_text is None: - findings.append( - ( - "DRIFT", - f"cspell: could not read {ws_name} on {ground} to check for a duplicated word list; verify by hand", - ) - ) - elif workspace_cspell_words(ws_text): - findings.append( - ( - "LETTER", - f"cspell: {ws_name} carries a cSpell word list while cspell.json is the single source of truth - delete the workspace copy (CODESTYLE.md Markdown and Spelling)", - ) - ) - - # --- Registry driftNotes freshness --- - findings.extend(driftnote_findings(entry, spec, len(findings))) - # Stamp the commit actually read for the ground-truth branch. - # Never fall back to another branch, since a stamp naming develop while carrying main's sha would misattribute every finding. - audited_sha = ground_head.get("commit", {}).get("sha", "") - return findings, audited_sha +def check_registry_driftnotes(run): + return driftnote_findings(run.entry, run.spec, len(run.findings)) _IS_WINDOWS = sys.platform == "win32" diff --git a/spec/checks.json b/spec/checks.json new file mode 100644 index 000000000..215d0c021 --- /dev/null +++ b/spec/checks.json @@ -0,0 +1,38 @@ +{ + "description": "The checks spec/audit.py runs per repository, in the order it runs them. Each id names the function check_ with every dot and hyphen as an underscore. scope repo runs once, baseline reads each baseline file of spec/files.json, and file runs on each baseline file that baseline read. hub false skips the hub's own audit. ground false still runs when the ground-truth branch is absent. suppression marks a check a driftNote naming its id suppresses, until the repository declares endsWithType, or on every repository where that is null.", + "checks": [ + { "id": "branch.facts", "family": "branch", "description": "main exists, and the registry's hasDevelop matches whether develop does.", "spec": "registry/repos.json", "hub": true, "scope": "repo", "ground": false }, + { "id": "branch.drift", "family": "branch", "description": "Paths changed on main that develop is behind on or has diverged from.", "spec": "registry/repos.json", "hub": true, "scope": "repo" }, + { "id": "settings.general", "family": "settings", "description": "The repository's general settings match the fleet's.", "spec": "repo-config/settings.json", "hub": true, "scope": "repo" }, + { "id": "rulesets.payload", "family": "rulesets", "description": "The main and develop rulesets match the workflow model's payloads, with no stray ruleset.", "spec": "repo-config/main.json", "hub": true, "scope": "repo" }, + { "id": "python.directories.declared", "family": "python", "description": "A tree carrying .py files declares the python type.", "spec": "spec/type-model.md", "hub": true, "scope": "repo", "suppression": { "endsWithType": "python" } }, + { "id": "python.directories.coverage", "family": "python", "description": "Every tracked .py file sits inside a directory the registry declares.", "spec": "registry/repos.json", "hub": true, "scope": "repo" }, + { "id": "python.directories.root", "family": "python", "description": "A root tests/ suite has the manifest the undeclared root default needs to run it.", "spec": "registry/repos.json", "hub": true, "scope": "repo" }, + { "id": "filenaming.scripts", "family": "filenaming", "description": "Each script file name takes the stem its extension sets.", "spec": "CODESTYLE.md", "hub": true, "scope": "repo", "suppression": { "endsWithType": null } }, + { "id": "secrets.stems", "family": "secrets", "description": "Each runtime secret file's stem is lowercase snake_case.", "spec": "GOVERNANCE.md", "hub": true, "scope": "repo" }, + { "id": "secrets.names", "family": "secrets", "description": "The secret names each store holds against those the applicable mechanisms require and forbid.", "spec": "spec/secrets.json", "hub": true, "scope": "repo" }, + { "id": "dependabot.ecosystems", "family": "dependabot", "description": "dependabot.yml declares each ecosystem the tree implies.", "spec": "AUDIT.md", "hub": true, "scope": "repo" }, + { "id": "files.presence", "family": "files", "description": "Each baseline file, or a declared alternative, is present, with its intent staleness advisory.", "spec": "spec/files.json", "hub": true, "scope": "baseline" }, + { "id": "interface.contract", "family": "interface", "description": "An interface workflow keeps its contract's job keys, check name, and tokens.", "spec": "spec/files.json", "hub": true, "scope": "file" }, + { "id": "interface.python-directories", "family": "interface", "description": "A caller's python-directories input matches the registry's pythonDirectories.", "spec": "registry/repos.json", "hub": true, "scope": "file" }, + { "id": "interface.concurrency", "family": "interface", "description": "The publisher's and the merge-bot's concurrency group is the one WORKFLOW.md pins.", "spec": "WORKFLOW.md", "hub": true, "scope": "file" }, + { "id": "verbatim.jobs", "family": "verbatim", "description": "Each job a contract pins verbatim matches the hub canonical.", "spec": "spec/files.json", "hub": true, "scope": "file" }, + { "id": "verbatim.file", "family": "verbatim", "description": "A verbatim file matches the hub canonical.", "spec": "spec/files.json", "hub": true, "scope": "file" }, + { "id": "sections.presence", "family": "sections", "description": "Each section a Markdown file owes is present as a heading.", "spec": "spec/files.json", "hub": true, "scope": "file" }, + { "id": "sections.verbatim", "family": "sections", "description": "Each verbatim section matches the hub canonical.", "spec": "spec/files.json", "hub": true, "scope": "file" }, + { "id": "sections.undeclared", "family": "sections", "description": "A carried instruction file holds no level-two heading the manifest does not declare.", "spec": "spec/section-model.md", "hub": false, "scope": "file" }, + { "id": "carried.template-ref", "family": "carried", "description": "A carried instruction file names the hub only inside its verbatim sections.", "spec": "GOVERNANCE.md", "hub": false, "scope": "file" }, + { "id": "carried.version-literal", "family": "carried", "description": "An instruction document carries no version literal or commit SHA.", "spec": "GOVERNANCE.md", "hub": true, "scope": "file" }, + { "id": "tree.readable", "family": "tree", "description": "The repository's file tree was read in full.", "spec": "spec/files.json", "hub": true, "scope": "repo" }, + { "id": "tree.verbatim", "family": "tree", "description": "Each manifest-owned tree matches its hub source.", "spec": "spec/files.json", "hub": false, "scope": "repo" }, + { "id": "hubonly.carried", "family": "hubonly", "description": "The repository carries no hub-hosted file the manifest leaves undeclared.", "spec": "spec/divergences.json", "hub": false, "scope": "repo" }, + { "id": "history.mirror", "family": "history", "description": "HISTORY.md opens with the README's title and tagline.", "spec": "spec/readme-structure.md", "hub": true, "scope": "repo" }, + { "id": "readme.description", "family": "readme", "description": "The README title and tagline, and the About and Docker Hub descriptions they mirror to.", "spec": "spec/readme-structure.md", "hub": true, "scope": "repo" }, + { "id": "readme.sections", "family": "readme", "description": "The README's sections, in order.", "spec": "spec/readme-sections.json", "hub": true, "scope": "repo" }, + { "id": "readme.shields", "family": "readme", "description": "The README's shields, by class and placement.", "spec": "spec/readme-sections.json", "hub": true, "scope": "repo" }, + { "id": "readme.links", "family": "readme", "description": "The README's reference links, by name, group, and order.", "spec": "spec/readme-sections.json", "hub": true, "scope": "repo" }, + { "id": "readme.tools", "family": "readme", "description": "Each cataloged tool the README lists carries the catalog's link and description.", "spec": "spec/third-party-tools.json", "hub": true, "scope": "repo" }, + { "id": "cspell.workspace", "family": "cspell", "description": "No workspace file duplicates cspell.json's word list.", "spec": "CODESTYLE.md", "hub": true, "scope": "repo" }, + { "id": "registry.driftnotes", "family": "registry", "description": "Each driftNote's check id and pending claim, read against the findings before it.", "spec": "registry/repos.json", "hub": true, "scope": "repo" } + ] +} diff --git a/spec/validate.py b/spec/validate.py index d31a59e58..9a31de633 100755 --- a/spec/validate.py +++ b/spec/validate.py @@ -117,6 +117,9 @@ "verbatimJobs", } +# A driftNote names a check as a parenthesized id, "(hugo.generator.pinned)", anywhere in the note. +CHECK_ID_RE = re.compile(r"\(([a-z][a-z0-9-]*(?:\.[a-z0-9-]+)+)\)") +CHECK_SCOPES = ("repo", "baseline", "file") MARKDOWN_INLINE_LINK = re.compile(r"\]\((?P[^)\s]+)") MARKDOWN_REFERENCE_LINK = re.compile(r"^\[[^]]+\]:\s*(?P\S+)", re.MULTILINE) TEMPLATE_REPOSITORY_URL = "https://github.com/ptr727/ProjectTemplate" @@ -386,6 +389,55 @@ def ground_truth_branch_errors_for_repo(repo, name): return [] +def check_registry_errors(registry, types, repos): + """spec/checks.json's shape, and every check id a driftNote names resolving against it or the catalog.""" + checks = registry.get("checks") if isinstance(registry, dict) else None + if not isinstance(checks, list): + return ["checks.json: 'checks' must be an array"] + errors = [] + ids = set() + for i, check in enumerate(checks): + where = f"checks.json: checks[{i}]" + if not isinstance(check, dict): + errors.append(f"{where} must be an object") + continue + cid = check.get("id") + if not isinstance(cid, str) or not CHECK_ID_RE.fullmatch(f"({cid})"): + errors.append(f"{where} id {cid!r} is not a dotted lowercase check id") + elif cid in ids: + errors.append(f"{where} id '{cid}' is declared more than once") + ids.add(cid) + for key in ("family", "description", "spec"): + value = check.get(key) + if not isinstance(value, str) or not value.strip() or "\n" in value: + errors.append(f"{where} {key} must be a one-line, non-empty string") + if isinstance(check.get("spec"), str) and not (ROOT / check["spec"]).is_file(): + errors.append(f"{where} spec '{check['spec']}' is not a file in this repository") + if not isinstance(check.get("hub"), bool): + errors.append(f"{where} hub must be true or false") + if check.get("scope") not in CHECK_SCOPES: + errors.append(f"{where} scope must be one of {', '.join(CHECK_SCOPES)}") + if not isinstance(check.get("ground", True), bool): + errors.append(f"{where} ground must be true or false") + suppression = check.get("suppression") + ending = suppression.get("endsWithType", 0) if isinstance(suppression, dict) else 0 + if "suppression" in check and not (ending is None or ending in types.get("types", {})): + errors.append(f"{where} suppression.endsWithType must name a project type or be null") + catalog = {c.get("id") for t in types.get("types", {}).values() for c in t.get("checks", [])} + catalog |= { + c.get("id") for d in types.get("crossCutting", {}).values() for c in d.get("checks", []) + } + for repo in repos.get("repos", []): + notes = repo.get("driftNotes") if isinstance(repo, dict) else None + for note in notes if isinstance(notes, list) else []: + for cid in CHECK_ID_RE.findall(note if isinstance(note, str) else ""): + if cid not in ids | catalog: + errors.append( + f"{repo.get('name')}: driftNote names check '{cid}', which neither spec/checks.json nor spec/project-types.json defines" + ) + return errors + + def python_directories_errors_for_repo(repo, name): """Shape errors for a registry entry's optional `pythonDirectories` (a repo declaring none passes none). @@ -1552,6 +1604,8 @@ def check_selector(where, applies_to): errors.append(f"divergences.json: gap '{gp}' tracking must be a string or null") errors.extend(investigate_tracking_errors(f"gap '{gp}'", g)) + errors.extend(check_registry_errors(load("spec/checks.json"), types, repos)) + if errors: print("Spec validation FAILED:") for err in errors: diff --git a/tests/test_audit_helpers.py b/tests/test_audit_helpers.py index 25bfdefd6..488ba6303 100755 --- a/tests/test_audit_helpers.py +++ b/tests/test_audit_helpers.py @@ -4,6 +4,7 @@ from __future__ import annotations import contextlib +import inspect import io import locale import pathlib @@ -1626,7 +1627,7 @@ def test_parse_args(self) -> None: ["hugo"], ["Theme record (hugo.vendored.provenence)."], 0, - ["does not define"], + ["neither spec/checks.json nor spec/project-types.json defines"], ), # A version string in parentheses is the false positive the id shape has to exclude. ( @@ -4313,5 +4314,36 @@ def test_intent_canonical_rel(self) -> None: self.assertEqual(audit.intent_canonical_rel(item, path), want) +class RegistryCase(unittest.TestCase): + """spec/checks.json and the check functions audit_repo calls from it.""" + + def test_every_registered_check_names_a_function_of_its_scope(self) -> None: + for check in audit.registered_checks(): + with self.subTest(check["id"]): + arity = len(inspect.signature(audit.check_function(check)).parameters) + self.assertEqual(arity, 1 if check["scope"] == "repo" else 2) + + def test_every_check_function_is_registered(self) -> None: + named = {audit.check_function(c).__name__ for c in audit.registered_checks()} + defined = {n for n in vars(audit) if n.startswith("check_") and callable(vars(audit)[n])} + defined -= {"check_function", "check_id_owner", "check_interface", "check_verbatim"} + defined -= {"check_intent_staleness"} + self.assertEqual(defined - named, set()) + + def test_suppression_ids_resolve_against_the_registry(self) -> None: + typed = {"types": ["docs"], "driftNotes": []} + self.assertTrue(audit.is_suppression_id(typed, "python.directories.declared")) + self.assertFalse( + audit.is_suppression_id({"types": ["python"]}, "python.directories.declared") + ) + self.assertTrue(audit.is_suppression_id({"types": ["python"]}, "filenaming.scripts")) + self.assertFalse(audit.is_suppression_id(typed, "settings.general")) + + def test_a_registry_id_is_a_known_check_id(self) -> None: + spec: dict[str, Any] = {"types": {"types": {}, "crossCutting": {}}} + self.assertEqual(audit.check_id_owner(spec, "settings.general"), (None, True)) + self.assertEqual(audit.check_id_owner(spec, "no.such-check"), (None, False)) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_spec_validate.py b/tests/test_spec_validate.py index c725d697f..e173ea48f 100755 --- a/tests/test_spec_validate.py +++ b/tests/test_spec_validate.py @@ -20,6 +20,67 @@ import audit import validate +CHECK_TYPES = {"types": {"python": {"checks": [{"id": "python.typed"}]}}, "crossCutting": {}} + + +class CheckRegistryCase(unittest.TestCase): + """spec/checks.json's shape and the driftNote ids that must resolve against it.""" + + def check(self, **fields: object) -> dict[str, object]: + base: dict[str, object] = { + "id": "a.b", + "family": "a", + "description": "One line.", + "spec": "spec/checks.json", + "hub": True, + "scope": "repo", + } + return {**base, **fields} + + def errors(self, checks: list[object], notes: tuple[str, ...] = ()) -> list[str]: + repos = {"repos": [{"name": "Widget", "driftNotes": list(notes)}]} + return validate.check_registry_errors({"checks": checks}, CHECK_TYPES, repos) + + def test_the_shipped_registry_is_clean(self) -> None: + self.assertEqual( + validate.check_registry_errors( + validate.load("spec/checks.json"), + validate.load("spec/project-types.json"), + validate.load("registry/repos.json"), + ), + [], + ) + + def test_each_malformed_field_is_reported(self) -> None: + cases: list[tuple[str, list[object], str]] = [ + ("a duplicate id", [self.check(), self.check()], "more than once"), + ("an undotted id", [self.check(id="flat")], "dotted lowercase"), + ("a two-line description", [self.check(description="a\nb")], "one-line"), + ("a spec that is no file", [self.check(spec="spec/none.json")], "not a file"), + ("a string hub flag", [self.check(hub="yes")], "hub must be"), + ("an unknown scope", [self.check(scope="tree")], "scope must be"), + ("a string ground flag", [self.check(ground="no")], "ground must be"), + ( + "a suppression naming no type", + [self.check(suppression={"endsWithType": "rust"})], + "endsWithType", + ), + ("a check that is no object", ["a.b"], "must be an object"), + ] + for label, checks, needle in cases: + with self.subTest(label): + errors = self.errors(checks) + self.assertEqual(len(errors), 1, errors) + self.assertIn(needle, errors[0]) + + def test_a_driftnote_id_resolves_against_the_registry_or_the_catalog(self) -> None: + notes = ("Vendored (a.b).", "Typed (python.typed).", "Unknown (no.such-check).") + errors = self.errors( + [self.check(), self.check(id="c.d", suppression={"endsWithType": None})], notes + ) + self.assertEqual(len(errors), 1, errors) + self.assertIn("'no.such-check'", errors[0]) + class CarriedRelativeLinkCase(unittest.TestCase): """A hub-valid relative link must also resolve after its section is carried.""" From 25c834ef8b12cd41f0af6e6c650b9114ee1ec837 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 23 Oct 2026 15:05:57 -0700 Subject: [PATCH 04/20] Trim the Audit's Docstrings and Split Out Its Markdown Checks Each docstring keeps what a caller needs: what the function returns, what None or an empty result means, and what it raises. The Markdown reading and the README checks move to spec/audit_markdown.py, which depends on nothing else in the audit. A size test holds spec/audit.py under 3,000 lines and the new module under 1,000. --- spec/audit.py | 1426 +++-------------------------------- spec/audit_markdown.py | 654 ++++++++++++++++ tests/test_audit_helpers.py | 69 +- tests/test_audit_size.py | 24 + 4 files changed, 822 insertions(+), 1351 deletions(-) create mode 100644 spec/audit_markdown.py create mode 100755 tests/test_audit_size.py diff --git a/spec/audit.py b/spec/audit.py index 3dab49511..2ffa4f5b6 100755 --- a/spec/audit.py +++ b/spec/audit.py @@ -3,35 +3,15 @@ # requires-python = ">=3.13" # dependencies = ["markdown-it-py>=4", "pyyaml>=6"] # /// -"""Live fleet audit: the deterministic subset of AUDIT.md, run from the hub (gh plus the libraries above). - -Compares each cataloged registry repo against the ground truth in this repo - general settings -(repo-config/settings.json), branch rulesets (normalized diff vs the model's payloads), secret -names (spec/secrets.json; values are never read), baseline/per-type file presence and per-scope -Markdown section presence on the ground-truth branch (spec/files.json, spec/scope-model.md), -hub-hosted files a repo carries and should delete (git-tracked here and undeclared in the manifest, -triaged by spec/divergences.json), script file names and secret-file stems against CODESTYLE.md -"Script and Utility File Names", intent-staleness advisories (a carried intent file whose hub -canonical changed after the copy last did), and branch-model facts (main/develop existence, develop -behind main), and a fleet-wide membership check (every non-fork repo the registry owner has on -GitHub carries a registry/repos.json entry, and an 'archived' entry's status agrees with GitHub's -own archived flag. Owner-initiated: run it when -onboarding a repo, when drift is suspected, or before fleet-wide changes. Read-only - it never -modifies a target. - -Findings: DEFECT (an applicable check fails outright), LETTER (a required file is absent - intent -unverified, judge per AUDIT.md section 7), DRIFT (non-breaking divergence, e.g. main-side -changes develop lacks, a stale secret, a registry field contradicting reality), ERROR (a gh call -failed, so the repo could not be fully audited). Exits non-zero when any repo has a DEFECT, -LETTER, or ERROR finding. - -Usage: uv run --no-project --script spec/audit.py [RepoName ...] [--branch REF] (default: every cataloged repo, -each read at its registry groundTruthBranch). --branch overrides that branch for the run, so a -convergence can be verified before it is promoted, without editing the registry. - -A shallow hub clone cannot answer the stale-vs-modified classification or the intent staleness -advisory. A sweep then reports one ERROR per repo and exits non-zero. -`--issue` exits 2 with the error on stderr. The error says: Run `git fetch --unshallow origin` in that checkout. +"""Live fleet audit: the deterministic subset of AUDIT.md, run from the hub with gh and the libraries above. + +Compares each cataloged registry repo against the ground truth in this repo and never modifies a target. +Findings are DEFECT, LETTER (a required file is absent, intent unverified), DRIFT (non-breaking divergence), and ERROR +(a gh call failed). It exits non-zero when any repo has a DEFECT, LETTER, or ERROR finding. + +Usage: uv run --no-project --script spec/audit.py [RepoName ...] [--branch REF] +--branch overrides each repo's registry groundTruthBranch for the run. +A shallow hub clone makes a sweep report one ERROR per repo, and `--issue` exits 2 with "Run `git fetch --unshallow origin` in that checkout." """ import argparse @@ -53,9 +33,20 @@ from datetime import UTC, datetime from typing import Any -import markdown_it import validate # sibling, import-safe (its main is guarded) import yaml +from audit_markdown import ( + description_findings, + heading_texts, + readme_link_findings, + readme_section_findings, + readme_shield_findings, + strip_md_links, + tagline, + third_party_tool_findings, + title_and_intro, + undeclared_h2_headings, +) from carried_text import extract_section, normalize, strip_sections from git_env import git_location_cleared @@ -72,16 +63,8 @@ "delete_branch_on_merge", ] # The bypass_actors field is deliberately absent from this list. -# Who may bypass a ruleset is a per-repository human decision taken in the UI. -# The configure.sh script in repo-config treats it that way, since apply writes the live list back unchanged and check reports it without asserting. -# Comparing it here would contradict that, and did. -# Once the payloads stopped declaring a bypass list, every repo whose live ruleset still had one reported a ruleset DEFECT. -# That was a field the fleet config had deliberately stopped managing. -# Two tools comparing one field under opposite policies is the defect, rather than the field's value. RULESET_SUBSET = ["name", "target", "enforcement", "conditions", "rules"] # Phrases that assert work is still outstanding, matched against a registry driftNote. -# Deliberately specific, so a note recording a permanent deviation must not match. -# An example of one that must not match is a note reading that there is no get-version-task and validate-task is relied on instead. PENDING_MARKERS = ["pending", "not yet", "owed", "todo", "still", "behind", "missing", "absent"] @@ -91,29 +74,12 @@ def load(rel): @functools.cache def hub_tracked(rev=None): - """The hub's own tracked paths at the resolved `main` commit, which is what "hub-side" means - for the hub-only comparison. - - `git ls-tree -r` at that commit rather than `git ls-files` against ROOT's checked-out index - (a filesystem walk is avoided too, since it would pick up __pycache__ and a local .venv and - make the result depend on working-tree state), so a path present on `main` but absent on - `develop`, or the reverse, is not silently missed or falsely added. - Filtered to regular-file modes (100644, 100755) the same - way `_git_revisions()` is: a directory, a symlink, or a submodule gitlink has no file content - to compare, and every caller here assumes a plain file at each returned path. `-z` NUL-delimits - the output so an unusual path is not C-quoted, which would otherwise return an escaped string - that matches nothing a caller compares it against. - - `rev` defaults to `_hub_main_rev()`. The --selftest fixture passes an explicit `rev` to check - against ROOT's own real tracked files without a network fetch, keeping the offline engine - self-test offline. - - A non-zero exit raises rather than returning an empty set, which would read as "the hub tracks - nothing" and silently clear every hub-only finding. + """The hub's own tracked paths at the resolved `main` commit. + + A git failure raises, since an empty set would clear every hub-only finding. """ walk_rev = _hub_main_rev() if rev is None else rev # No text=True: -z's pathnames are raw bytes, and locale decoding here would crash on an invalid byte before the NUL-split below ever runs. - # Decode each record with a fixed utf-8/surrogateescape policy instead of os.fsdecode(), whose error handler is platform-dependent (surrogatepass on Windows) and still crashes on an arbitrary invalid byte there. r = subprocess.run( ["git", "ls-tree", "-r", "-z", walk_rev], cwd=ROOT, @@ -137,22 +103,7 @@ def hub_tracked(rev=None): def hub_only_paths(spec, rev=None): - """Hub-tracked paths the manifest does not declare, so a downstream copy is hub-hosted content rather than a carry. - - This is the deletion detector: the manifest says what a repo carries, so a file the hub tracks and the - manifest omits is the hub's own, and a downstream copy of one is drift whose remedy is a deletion. - Deriving it this way rather than from a hand-kept retirement list means a file dropped from the manifest - starts being detected on the next run, with no second place to remember to edit. - - A hit is a candidate rather than a verdict, because the match is on path alone. - A repo whose own content sits at a path the hub also uses matches without carrying anything of the hub's, - which the first fleet run showed twice: a KiCad tooling doc at scripts/README.md, and per-repo formatting - hooks at .husky/pre-commit. - So only a `retire` disposition in spec/divergences.json asserts a deletion, and an untriaged hit asks for - the file to be read. - - `rev` is passed through to `hub_tracked()`. See its docstring. - """ + """Hub-tracked paths the manifest does not declare, so a downstream copy is hub-hosted content rather than a carry.""" declared = {e["path"] for e in spec["files"]["baseline"]} tree_paths = set() for declaration in spec["files"].get("trees", []): @@ -167,11 +118,7 @@ def hub_only_paths(spec, rev=None): def gap_dispositions(spec): - """The curated ledger's manifest-gap dispositions, as path -> (disposition, reason). - - A gap the ledger triages is judged by its disposition rather than reported raw, so an accepted one - (every repo owns its LICENSE) is not a finding and a retired one names the deletion. - """ + """The curated ledger's manifest-gap dispositions, as path -> (disposition, reason).""" out = {} for g in spec.get("divergences", {}).get("gaps", []): if ( @@ -184,13 +131,7 @@ def gap_dispositions(spec): def repo_tree_blobs(slug, ground_head): - """Every blob path on a repo branch mapped to its object SHA and file mode, or None when it cannot be read in full. - - The trees endpoint takes a tree sha rather than a ref name, so the branch payload already read is - resolved to its tree instead of passing the branch name, which 404s. - A truncated response returns None rather than a partial set, since a partial tree cannot distinguish a - file the repo does not carry from one the response stopped short of. - """ + """Every blob path on a repo branch mapped to its object SHA and file mode, or None when it cannot be read in full.""" sha = (((ground_head or {}).get("commit") or {}).get("commit") or {}).get("tree", {}).get("sha") if not sha: return None @@ -229,11 +170,7 @@ def is_python_test_module(path): def is_csharp_test_project(path): - """Whether path is a C# project either named or located as a test project. - - Read two ways because lowercasing the whole basename would claim Contest.csproj and - Latest.csproj, which carry those letters inside a word rather than as a name segment. - """ + """Whether path is a C# project either named or located as a test project.""" if not path.endswith(".csproj"): return False name = path.rsplit("/", 1)[-1] @@ -246,11 +183,7 @@ def is_csharp_test_project(path): def coverage_candidate_types(types, repo_profiles): - """The declared types whose profile leaves them owing coverage at all, before tests are looked for. - - Python's lint-only profile still runs its suite under coverage, so only a lint-only language other - than Python drops out here, the type model holding coverage N/A for it. - """ + """The declared types whose profile leaves them owing coverage at all, before tests are looked for.""" profiles = repo_profiles if isinstance(repo_profiles, dict) else {} return [t for t in types if t == "python" or profiles.get(t) != "lint-only"] @@ -258,22 +191,7 @@ def coverage_candidate_types(types, repo_profiles): def coverage_claiming_types(types, type_mechanisms, tree): """The declared types that owe Codecov coverage, meaning the CODECOV_TOKEN secret and codecov.yml. - A type owes coverage when the fleet maps it to the codecov mechanism and the repo carries tests for - it, `types` having already passed coverage_candidate_types(). The test condition is what keeps a package-only build repo, a library - whose tests live elsewhere or are not yet written, from being told to store a token and commit a - codecov.yml whose statuses gate a report its pipeline never produces. - - Each detector is deliberately broader than the hub validator's own guard, which keys its C# leg on - `**/*Tests*.csproj` and its Python leg on a `tests/` directory in each Python directory it gates. - D1.6 owes coverage to every repo with tests, and a repo the hub leg does not reach meets it through - its own workflows instead, so a detector matching the guard exactly would drop the claim for a repo - whose suite sits in `test/` or whose project is named `App.UnitTest.csproj`. Every uncertainty here - therefore resolves toward keeping the claim, since a claim kept wrongly surfaces as a finding a - reader can dismiss and a claim dropped wrongly is a silent clean report on a repo nobody measured. - - `tree` is the repo's blob path set, or None when it could not be read in full. On None the test - question is unanswerable, so every candidate type keeps its claim, for that same reason. A type - mapped to codecov with no detector here keeps its claim too. + `tree` is the blob path set, or None when unread, and then every candidate keeps its claim. """ detectors = { "csharp": lambda paths: any(is_csharp_test_project(p) for p in paths), @@ -294,26 +212,21 @@ def coverage_claiming_types(types, type_mechanisms, tree): def python_directories_of(entry): """The registry's declared `pythonDirectories` for entry, or [] where none is declared. - Read defensively (a non-list, or a non-string element, drops rather than raises) since a malformed - declaration is spec/validate.py's own finding, and this audit gate must not crash mid-run on a - registry CI already gates on push. + A malformed declaration drops rather than raises, since spec/validate.py reports it. """ dirs = entry.get("pythonDirectories") return [d for d in dirs if isinstance(d, str)] if isinstance(dirs, list) else [] def python_directory_covers(directory, path): - """Mirrors .github/actions/python-directories/python_directories.py's covers(): '.' covers every - path, and any other declared directory covers only its own subtree.""" + """Mirrors python_directories.py's covers(): '.' covers every path, any other directory its own subtree.""" return directory == "." or path.startswith(directory + "/") def expected_python_directories(entry, tree): """The directories a caller's `python-directories` declaration is expected to resolve to. - Mirrors python_directories.py's resolve() fallback for a caller declaring none: the registry's own - `pythonDirectories` where declared, else the root where the tree carries a root pyproject.toml, else - no directory at all. `tree` is the repo's blob path set, or None when it could not be read in full. + `tree` is the blob path set, or None when unread. """ declared = python_directories_of(entry) if declared: @@ -324,16 +237,11 @@ def expected_python_directories(entry, tree): # The catalog's hook snippets carry this helper into repos of every language, so it is fleet tooling rather than the repo's own Python. -# A repo that copied it before the rename keeps the hyphenated name until it is next resynced. CARRIED_PYTHON_HELPERS = {"hub_fetch_run.py", "hub-fetch-run.py"} def declared_types(entry): - """The string elements of the registry entry's `types` list, or an empty list where `types` is not a list. - - A malformed entry therefore reads as untyped, or as typed by its string elements alone, rather than - aborting the run. - """ + """The string elements of the registry entry's `types` list, or an empty list where `types` is not a list.""" types = entry.get("types") return [t for t in types if isinstance(t, str)] if isinstance(types, list) else [] @@ -355,11 +263,7 @@ def quote_note(note): def python_type_findings(entry, tree): """The discovery advisory for tracked `.py` files while the registry's `types` omits 'python'. - Advisory rather than a defect, per spec/type-model.md's undeclared-but-detected row: the repo - declares the type, or suppresses the advisory with a driftNote naming - (python.directories.declared) and its reason, for vendored or incidental Python. `tree` is the - repo's blob path set, or None when it could not be read in full, in which case Python's presence - is unanswerable and nothing is reported. + `tree` is the blob path set, or None when unread, which reports nothing. """ if tree is None or "python" in declared_types(entry): return [] @@ -395,12 +299,7 @@ def python_type_findings(entry, tree): def python_undeclared_root_findings(entry, tree): """DRIFT where a registry declaring no Python directory leaves a root suite no gate runs. - The validator's root default runs a root tests/ suite only beside a root uv.lock or - requirements*.txt, the gate it ran before directories could be declared. A root suite with - neither runs only once a lint-only root is declared or a [project] root gains a manifest, and - the blob paths alone cannot say which the root is. Meanwhile coverage_claiming_types() still - claims coverage for it. `tree` is the repo's blob path set, or None when it could not be read - in full, in which case nothing is reported. + `tree` is the blob path set, or None when unread, which reports nothing. """ if tree is None or "python" not in declared_types(entry): return [] @@ -427,15 +326,9 @@ def python_undeclared_root_findings(entry, tree): def python_directory_coverage_findings(entry, tree): - """DRIFT naming tracked `.py` files that sit outside every directory expected_python_directories() - expects, mirroring python_directories.py's own uncovered() but measured against the registry's - declaration rather than a caller's own runtime one, so a registry that stops matching what a repo's - callers actually pass is caught here rather than only in each caller's own warning annotation. - - `tree` is the repo's blob path set, or None when it could not be read in full, in which case coverage - is unanswerable and nothing is reported. The finding names the count and up to 5 example paths, since - a whole tree of them would bury the message the same way python_directories.py's own warning caps at - UNCOVERED_SHOWN rather than listing every path. + """DRIFT naming tracked `.py` files outside every directory the registry's declaration expects. + + `tree` is the blob path set, or None when unread, which reports nothing. """ if tree is None or "python" not in declared_types(entry): return [] @@ -480,26 +373,8 @@ def capped_paths(paths): def file_naming_findings(entry, blobs, read_text): """DRIFT for script file names that break CODESTYLE.md "Script and Utility File Names". - `blobs` maps each blob path to its file mode. It is None when the tree could not be read in - full. Nothing is reported then, since the run's one `tree:` finding names that case. - `read_text` returns a path's text, or None where none is returned. It is called only for a - non-conforming extensionless executable, whose shebang decides whether the rule covers it. - - A `.py` stem is lowercase snake_case. A `.sh`, `.ps1`, `.cmd`, or `.bat` stem is lowercase- - hyphenated once Pester's `.Tests.ps1` suffix is set aside, and so is an extensionless - executable's name. An extension is recognized whatever its case, so an upper-case one breaks. - A name whose text after its last dot is none of those extensions is not judged, whatever its - mode. So an executable dotfile with no such extension, and a dotted name such as `Run_Me.v2`, - go unjudged. - A snake_case launcher with an extension, beside a `.py` of the same stem in its own directory, - keeps that stem. Three cases go to judgment rather than breaking. One is a launcher whose `.py` - sits only in another directory. Another is an extensionless launcher beside its `.py`, which - may be an installed command instead. The last is an extensionless executable with no shebang - or readable text. - - A registry driftNote naming (filenaming.scripts) exempts each path it names as a code span, for a - name a tool or platform fixes. One note therefore cannot silence a later break on another path. - A named path the check would not report is a stale exemption, and so is a note naming no path. + `blobs` maps each blob path to its mode, or is None when unread, which reports nothing. + `read_text` returns a path's text or None, and is called only for an extensionless executable. """ if blobs is None: return [] @@ -612,12 +487,7 @@ def file_naming_findings(entry, blobs, read_text): def secret_stem_findings(tree): """DRIFT naming each secret file in a repo-scoped `.secrets/` directory whose stem is not lowercase snake_case. - A secret file sits directly in the directory, as its `**/.secrets/*` ignore rule reads it. The - directory's own README.md, .gitignore, and .gitkeep are not secret files. The stem is the - name before its first dot, so `mail_relay.env.example` reads as conforming. Real secret - files are untracked, so their tracked `.example` files are what the tree shows of them. `tree` is - the repo's blob path set, or None when it could not be read in full. Nothing is reported then, - as in file_naming_findings(). + `tree` is the blob path set, or None when unread, which reports nothing. """ if tree is None: return [] @@ -644,11 +514,9 @@ def secret_stem_findings(tree): @functools.cache def canonical_blob_sha(path): - """The hub's git blob identity for path, from the same resolved `main` commit - `_git_revisions()` and `_hub_main_rev()` walk, not from ROOT's checked-out working tree, - which is not necessarily `main`. Raises OSError, matching - a filesystem read's own contract, when path is absent from that commit or is not a regular - file there. + """The hub's git blob identity for path at the resolved `main` commit, not the working tree. + + Raises OSError where path is absent from that commit or is not a regular file there. """ rev = _hub_main_rev() # Read via git ls-tree, not git rev-parse :, which resolves a directory to a tree object id just as readily as a file to a blob one, silently breaking the regular-file promise above. @@ -683,12 +551,7 @@ def tree_path_included(path, patterns): @functools.cache def hub_name(): - """This repo's name on the remote, and whether it came from the remote. - - Read from origin rather than the checkout directory, which a differently-named clone, a fork, or a - worktree without an origin would silently break. The caller announces the directory-name fallback: - a silently degraded match is the fail-open case this check exists to prevent. - """ + """This repo's name on the remote, and whether it came from the remote.""" r = subprocess.run( ["git", "config", "--get", "remote.origin.url"], capture_output=True, @@ -707,11 +570,7 @@ def hub_repo_name() -> str: def gh(path, ok404=False) -> Any: - """GET a REST path via gh, returning parsed JSON, or None on a 404 (when ok404) or an empty response body. - - No --paginate: on object endpoints it concatenates page documents into unparseable JSON. Every - list read here fits one page; callers pass per_page=100 where a default page could truncate. - """ + """GET a REST path via gh, returning parsed JSON, or None on a 404 (when ok404) or an empty response body.""" r = subprocess.run( ["gh", "api", path], capture_output=True, text=True, encoding="utf-8", check=False ) @@ -725,24 +584,10 @@ def gh(path, ok404=False) -> Any: def owner_repos(owner): - """Every non-fork repository the registry owner actually owns on GitHub, paginated by hand. - - /user/repos rather than /users/{owner}/repos: the latter returns only public repos unless gh - is authenticated as that exact user, which would silently under-count a private repo the same - way a missing registry entry does. Confirm the authenticated login matches owner first, so a - mismatch is a loud error rather than a quietly wrong (and possibly cross-account) result. - No --paginate, per the gh() docstring above. Page by hand until a page comes back short. - - This still assumes the credential gh runs as can see every repo the owner has: an ordinary - `gh auth login` session does, but a fine-grained PAT scoped to "selected repositories" would - return an incomplete list with no error and no signal in the response to detect that from, so - the sweep would read as clean while some repos were never inspected. Run this as the owner's - own account, not a repository-scoped credential. - """ + """Every non-fork repository the registry owner actually owns on GitHub, paginated by hand.""" me = gh("user") login = me.get("login") if isinstance(me, dict) else None # GitHub logins are case-insensitive. - # Compare lowered, or a registry owner spelled with different casing than gh's own login reads as a mismatch when it is the same account. if (login or "").lower() != owner.lower(): raise RuntimeError( f"gh is authenticated as '{login}', not registry owner '{owner}'. " @@ -752,8 +597,6 @@ def owner_repos(owner): while True: batch = gh(f"user/repos?affiliation=owner&per_page=100&page={page}") # An empty response body reads as None here, per the gh() docstring above. - # `or []` would silently take that for a short page and stop the sweep, reporting the fleet clean while the rest of the repos were never actually inspected. - # Fail loud instead, matching the ownership guard above. if batch is None: raise RuntimeError(f"GitHub returned an empty repository-list response (page {page})") if not isinstance(batch, list) or not all(isinstance(r, dict) for r in batch): @@ -765,21 +608,7 @@ def owner_repos(owner): def repo_identity(url): - """owner/repo, lowercased, parsed from a github.com URL, or None if it does not parse. - - The canonical identity two repos are compared by, since a bare repo name collides across owners. - GitHub's own full_name field is already this exact shape (case-preserved). - A trailing .git is stripped so it still resolves to the same identity full_name would. - A query string or fragment is rejected outright rather than folded into the repo name. - - spec/validate.py's github_identity() is the one parse, called rather than restated, since a byte-identical copy - of that regex here is a second definition that only looks like one. - - The padding is trimmed before that call, deliberately and in one direction only. This function decides whether a - live repo has an entry, so a padded url resolving to nothing here would report the repo as having no entry at all, - a DEFECT naming the wrong problem. The gate refuses the padding itself, which is where that finding belongs, so - audit reads the value as generously as it can and validate.py is what says no. - """ + """owner/repo, lowercased, parsed from a github.com URL, or None if it does not parse.""" if not isinstance(url, str): return None slug = validate.github_identity(url.strip()) @@ -787,24 +616,9 @@ def repo_identity(url): def membership_findings(spec): - """Fleet-wide: every non-fork repo the owner has on GitHub must have a registry entry. - - Nothing else in this file, or in spec/validate.py, ever looks past the registry to what - actually exists, so a repo that never got an entry is invisible to every tool that reads it. - The registry was treated as ground truth about existence, not just about conformance. Ownership only: a fork, or a repo the - owner merely collaborates on, was never meant to carry a registry entry. - - A registry entry's status then says what, if anything, the rest of the audit owes the repo: - 'cataloged' is audited normally, 'backlog' awaits classification (existing behavior), and - 'archived'/'excluded' are known and out of scope by design: archived because GitHub itself - says the repo takes no further work, excluded because exclusionReason records a human - decision that it should not be audited. Both still require an entry, so the decision stays - visible in the catalog instead of the repo just disappearing from it. - """ + """Fleet-wide: every non-fork repo the owner has on GitHub must have a registry entry.""" findings = [] # A bare-name match would treat a same-named repo under a different owner as this one's entry. - # Keyed by owner/repo instead, the shape GitHub's own full_name field already carries. - # Two entries resolving to the same identity are a separate, validate.py-level defect. registry_by_identity = {} for r in spec["registry"]["repos"]: identity = repo_identity(r.get("url")) if isinstance(r, dict) else None @@ -837,25 +651,6 @@ def membership_findings(spec): return findings -def docker_hub_description(slug): - """The Docker Hub short description for a repo, or None if the image is genuinely absent (HTTP 404). - - The image name is taken as owner/repo lowercased, the fleet convention (`ptr727/PhotoCleaner` -> - `ptr727/photocleaner`), so a repo whose image is named otherwise, or not yet pushed, 404s and is skipped - rather than falsely flagged. A transient failure (timeout, network, non-404 status) is **raised**, not - swallowed, so the caller surfaces "could not verify" instead of silently passing. Read-only, unauthenticated. - """ - owner, repo = slug.split("/", 1) - url = f"https://hub.docker.com/v2/repositories/{owner.lower()}/{repo.lower()}/" - try: - with urllib.request.urlopen(url, timeout=5) as r: - return json.loads(r.read().decode("utf-8")).get("description") - except urllib.error.HTTPError as e: - if e.code == 404: - return None - raise - - def normalize_ruleset(payload): sub = {k: payload.get(k) for k in RULESET_SUBSET} if isinstance(sub.get("rules"), list): @@ -888,19 +683,7 @@ def is_suppression_id(entry, cid): def driftnote_findings(entry, spec, open_count): - """Freshness findings over one repo's registry driftNotes, given how many findings the audit already has. - - Neither shape below is gated on the rest of the audit being clean, and the gate that used to wrap both - is the defect this replaces: one standing finding a repo cannot clear exempted its whole note list, so - the repo with open findings, where a stale note is most likely, was the one never checked. - - A note naming a check id declares what would retire it, so it is surfaced on every run. What the audit - cannot do is decide it, since no per-type check is mechanized, so the id is resolved against the - catalog here and the check itself is left to the auditor. A pending-marker note is a prose claim that - work is outstanding, so a clean audit contradicts it outright while an unclean audit only asks which of - the open findings it means. Narrow markers keep a permanent-deviation note ("relies on validate-task") - from tripping. - """ + """Freshness findings over one repo's registry driftNotes, given how many findings the audit already has.""" out = [] for note in entry.get("driftNotes", []): quoted = quote_note(note) @@ -951,40 +734,14 @@ def driftnote_findings(entry, spec, open_count): # The owner half of the slug repo_slug() answers with when it cannot parse an entry's url, plus the stand-in for a name it cannot use. -# Both carry an underscore, which GitHub allows in a repository name and not in an owner name. -# So the pair addresses no repository that can exist, while still naming the entry in every message the finding reaches. UNRESOLVED_OWNER = "_unresolved" UNNAMED_ENTRY = "_unnamed" def repo_slug(entry): - """The `/` every `repos/{slug}/...` read below addresses. - - Parsed by spec/validate.py's own url grammar rather than by taking the last two path segments, so what this - addresses and what that gate accepted are the same value. Taking the segments kept a trailing `.git` that the - gate strips for the identity, sending every read to `repos//.git/...`. - - A url this module cannot parse never raises, because this module is not the gate. spec/validate.py refuses such - a url, and nothing runs it before an audit: spec/fidelity_honesty.py and spec/workflow_reuse.py both call this on - every entry with no handler, so raising would abort a whole fleet report over one malformed entry that instead - 404s and lands in the unreadable bucket beside its healthy siblings. The finding stays where it belongs, on the - gate. - - Anything else answers with UNRESOLVED_OWNER and the entry's name, so every read 404s and the message names the - entry that caused it. spec/workflow_reuse.py reaches this by building `{"name": hub_repo_name()}` as its own - fallback when the hub has no registry entry and calling this on it. - - Each half of that sentinel answers a way a fallback slug has already addressed the wrong thing. It carries two - segments because a one-segment value shifts every later path component up one: `repos/Fixture/git/trees/` - reads owner `Fixture` and repository `git` rather than failing. Its owner segment carries an underscore, which - GitHub does not allow in an owner name, so it cannot collide with a real owner. And a name that is nothing but - dots is dropped rather than encoded, because GitHub decodes `%2e` and then normalizes the dot segment it becomes, - so `repos//../branches/main` would read `repos/branches/main`. - - Taking the url's last two path segments is what all of this replaced, and it produced a plausible slug rather - than a failing one. `https://gitlab.test/owner/Repo` became `owner/Repo` and read that repository on github.com, - and a `?` survived into the value, where `repos//?tab=readme/branches/main` ends its path at - `repos//` and sends the rest as a query string. + """The `/` every `repos/{slug}/...` read addresses. + + An unparseable url never raises, it answers with UNRESOLVED_OWNER and the entry's name. """ slug = validate.github_identity(entry.get("url")) if slug is not None: @@ -995,18 +752,11 @@ def repo_slug(entry): def repo_selectors(entry, defaults): - """The scope-selector set a files.json appliesTo is matched against (see spec/scope-model.md). - - The four namespaces - project types, workflowModel, releaseTrigger, consumerModel - are disjoint, so - a flat token set is unambiguous. workflowModel and releaseTrigger resolve repo -> defaults -> fleet - default (as configure.sh does). consumerModel has no fleet default - validate.py requires it on every - cataloged repo, so a cataloged repo always contributes one. - """ + """The scope-selector set a files.json appliesTo is matched against (see spec/scope-model.md).""" sel = set(declared_types(entry)) sel.add(entry.get("workflowModel") or defaults.get("workflowModel") or "release") sel.add(entry.get("releaseTrigger") or defaults.get("releaseTrigger") or "two-phase") # The consumerModel field has no defaults fallback, since the registry schema does not allow defaults.consumerModel and validate.py requires it on every cataloged repo. - # The guard only shields a malformed non-cataloged entry. cm = entry.get("consumerModel") if cm: sel.add(cm) @@ -1014,7 +764,7 @@ def repo_selectors(entry, defaults): def applies(applies_to, sel): - """True if an appliesTo selector applies to a repo's selector set. Disjunctive any-of, with `*` meaning all.""" + """True if an appliesTo selector applies to a repo's selector set.""" if applies_to == "*": return True tokens = applies_to if isinstance(applies_to, list) else [applies_to] @@ -1022,19 +772,14 @@ def applies(applies_to, sel): def _section_spec(elt): - """Normalize a sections[] entry to (name, appliesTo, fidelity). A bare string is appliesTo `*`, intent.""" + """Normalize a sections[] entry to (name, appliesTo, fidelity).""" if isinstance(elt, str): return elt, "*", "intent" return elt.get("name", ""), elt.get("appliesTo", "*"), elt.get("fidelity", "intent") def required_sections(item, sel): - """Section names to presence-check (heading grep) for this repo - the non-verbatim sections. - - A bare-string section is appliesTo `*`, intent; an object section carries its own selector and fidelity. - A verbatim section is checked byte-for-byte instead (verbatim_sections), so it is excluded here to avoid a - redundant presence finding. The entry's own appliesTo is assumed already matched by the caller. - """ + """Section names to presence-check (heading grep) for this repo, the non-verbatim sections.""" out = [] for elt in item.get("sections", []): name, sec, fid = _section_spec(elt) @@ -1044,7 +789,7 @@ def required_sections(item, sel): def verbatim_sections(item, sel): - """Section names marked fidelity verbatim for this repo - checked byte-for-byte against the hub canonical.""" + """Section names marked fidelity verbatim for this repo, checked byte-for-byte against the hub canonical.""" out = [] for elt in item.get("sections", []): name, sec, fid = _section_spec(elt) @@ -1064,812 +809,19 @@ def verbatim_sections(item, sel): def version_literals_outside_verbatim(text, verbatim_names): - """Every version literal or commit SHA in `text` outside its verbatim sections, sorted and deduplicated. - - A pin lives in the workflow or manifest that uses it, where Dependabot moves it, so a copy in prose is - stale at the next bump and sends an agent to edit governance for a change that needed none. - Verbatim sections are excised downstream for the reason template_ref_outside_verbatim gives: their - bytes are the hub's, so a literal there is the hub's defect to fix once rather than each repo's. The - hub's own run passes no sections, which is where that defect is caught. - Fenced code is scanned too, since a pinned `uses:` line in a sample goes stale the same way. - """ + """Every version literal or commit SHA in `text` outside its verbatim sections, sorted and deduplicated.""" return sorted( set(VERSION_LITERAL.findall(strip_sections(text, verbatim_names, keep_pins=True))) ) -def undeclared_h2_headings(text, declared): - """Level-two headings in `text` that `declared` does not name, folded and sorted.""" - h2s = {h.strip().lower() for level, h, _, _ in _atx_headings(text) if level == 2} - return sorted(h2s - {d.strip().lower() for d in declared}) - - def template_ref_outside_verbatim(text, verbatim_names, hub_name): - """True when `hub_name` appears in `text` outside every one of its verbatim sections. - - A verbatim section's bytes are the hub's canonical and are checked byte-for-byte elsewhere, so a hub - reference inside one cannot be removed downstream: the repo would have to fail the verbatim check to - clear this one. `AGENTS.md > Fleet Bootstrap` is the standing case, since naming the hub is that - section's entire function - it is the byte-locked entry point stating where the canonical rules live, - and a repo holding no current copy of anything else is exactly who reads it. Excising the verbatim - regions before scanning keeps the check pointed at the prose a repo actually owns. A hub reference that - reaches a verbatim section is the hub's defect to fix once in the canonical, never each repo's to clear. - """ + """True when `hub_name` appears in `text` outside every one of its verbatim sections.""" return hub_name.lower() in strip_sections(text, verbatim_names).lower() -# CommonMark plus tables, with escapes kept as their own tokens so a line can be rebuilt from its source. -_MD = markdown_it.MarkdownIt("commonmark").enable("table").disable("text_join") - - -def _atx_headings(text): - """`(level, raw text, first line, last line)` for each ATX heading, the form the region reader splits on.""" - tokens = _MD.parse(text) - return [ - (int(t.tag[1]), tokens[i + 1].content, t.map[0], t.map[1]) - for i, t in enumerate(tokens) - if t.type == "heading_open" and t.markup.startswith("#") and t.map - ] - - -def references(text): - """The reference definitions markdown-it reads from a document, keyed the way it resolves a link.""" - env = {} - _MD.parse(text, env) - return env.get("references", {}) - - -def heading_texts(markdown): - """Lowercased heading texts in a Markdown document, for case-insensitive section-presence matching.""" - return {h.strip().lower() for _, h, _, _ in _atx_headings(markdown)} - - -_HTML_COMMENT = re.compile(r"", re.DOTALL) - - -def strip_md_links(text, references=None): - """`text` with each Markdown link reduced to its label, everything else kept as written. - - `references` is the document's reference table, so `[text][ref]` is a link only where `ref` is defined. - """ - out = [] - for tok in _MD.parseInline(text, {"references": dict(references or {})})[0].children or []: - if tok.type in ("text", "html_inline"): - out.append(tok.content) - elif tok.type == "code_inline": - out.append(tok.markup + tok.content + tok.markup) - elif tok.type == "softbreak": - out.append("\n") - elif tok.type not in ("link_open", "link_close"): - out.append(tok.markup) - return "".join(out) - - -def title_and_intro(text): - """The H1 title text and the intro region between it and the first H2, HTML comments stripped.""" - lines = _HTML_COMMENT.sub("", normalize(text)).split("\n") - heads = _atx_headings("\n".join(lines)) - first = next((h for h in heads if h[0] == 1), None) - if first is None: - return None, "" - end = next((h[2] for h in heads if h[0] == 2 and h[2] > first[2]), len(lines)) - return first[1].strip(), "\n".join(ln.rstrip() for ln in lines[first[3] : end]).strip("\n") - - -def tagline(intro): - """The first line of a README/HISTORY intro region - the one canonical short description. - - Per spec/readme-structure.md item 1, only this line carries the length and link-free rules and only this - line mirrors to the About panel, the Docker Hub short description, and the HISTORY.md opening. Any further - paragraph is free prose no mirror reads, so comparing the whole region would report a legitimate second - paragraph as a mirror difference. - """ - return intro.split("\n")[0] - - -# Text scans, since a parsed link keeps its target but not the reference name and position these checks read. -_MD_IMAGE_REF = re.compile(r"!\[[^\]]*\]\[([^\]]+)\]") -_MD_IMAGE_INLINE = re.compile(r"!\[[^\]]*\]\((\S+?)\)") -_LINK_DEF = re.compile(r"^\[([^\]]+)\]:\s*(\S+)", re.MULTILINE) -# A URI scheme, requiring two or more characters so a `C:` drive letter is not read as one. -# Every scheme a README actually carries (mailto, ftp, ssh, git, tel, data) is longer than that. -_URI_SCHEME = re.compile(r"[a-z][a-z0-9+.\-]+:", re.IGNORECASE) - - -def unfenced_text(text): - """`text` normalized with every fenced code block's lines removed, since markup in a sample is shown rather than used.""" - lines = normalize(text).split("\n") - fenced = { - i for t in _MD.parse("\n".join(lines)) if t.type == "fence" and t.map for i in range(*t.map) - } - return "\n".join(ln for i, ln in enumerate(lines) if i not in fenced) - - -def readme_region(text, heading): - """extract_section against a comment-stripped copy of the document. - - extract_section matches a heading by its exact parsed text, so `## License ` is not - the section named License and the lookup returns None. Four repos suffix every heading that way for the - Markdown All in One extension, and reading them as sectionless reported twelve absent sub-sections that - were present. Stripping first is confined to the README checks: extract_section itself must keep the - comment, since the verbatim engine hashes a section's exact bytes. - - Fences are stripped here rather than at each caller. Every region the README checks read comes through - this one function, so doing it at the root is what stops a code sample inside a section from being read - as a badge, a tool row, or a definition by whichever caller forgot. - """ - return extract_section(_HTML_COMMENT.sub("", unfenced_text(text)), heading) - - -def shield_endpoints(region, defs): - """The image URLs a Markdown region renders, from both its `![alt][ref]` and its `![alt](url)` uses. - - Keyed on the endpoint rather than the alt text or the reference name, because those are captions. The - fleet writes Release Status, Releases Build, Build Status, Workflow Status and Lint Build for one badge, - and names its reference `last-commit-shield` and `lastcommit-shield` in the same breath, while the - endpoint under img.shields.io is identical in every repo. - - Inline uses are resolved as well as reference ones. Reading references alone made an inline shield - invisible rather than wrong, so a repo writing every badge inline, which the reference-link rule forbids - for a separate reason, would have passed the shield check by carrying nothing the check could see. - A `[ref]: url` definition line is not an image, so it is never counted as a use, which is what keeps a - shield's own definition from reading as a second placement of it. - """ - region = region or "" - urls = [defs[m.group(1)] for m in _MD_IMAGE_REF.finditer(region) if m.group(1) in defs] - return urls + [m.group(1) for m in _MD_IMAGE_INLINE.finditer(region)] - - -def shield_matches(url, shield): - """True where a rendered URL is the shield the model describes, by endpoint plus its query discriminators.""" - return ( - shield["match"] in url - and all(q in url for q in shield.get("requireQuery", [])) - and (shield.get("forbidQuery") is None or shield["forbidQuery"] not in url) - ) - - -def addressed_region(text, address): - """The region a shield's `in` address names, as `Heading` or `Heading > Sub-heading`, or None if absent.""" - parts = [p.strip() for p in address.split(">")] - body = readme_region(text, parts[0]) - if body is None or len(parts) == 1: - return body - lines = body.split("\n") - h3s = [h for h in _atx_headings(body) if h[0] == 3] - for i, (_, name, _, end) in enumerate(h3s): - if name.strip().lower() == parts[1].lower(): - stop = h3s[i + 1][2] if i + 1 < len(h3s) else len(lines) - return "\n".join(lines[end:stop]) - return None - - -def ordered_headings(markdown, level): - """Heading texts at `level` in document order, HTML comments stripped, fenced blocks skipped.""" - return [ - _HTML_COMMENT.sub("", h).strip() for lv, h, _, _ in _atx_headings(markdown) if lv == level - ] - - -def readme_section_findings(text, model, sel, public): - """README section presence and order against spec/readme-sections.json. - - Headings the model does not name are dropped before the order comparison, so the roughly sixty genuinely - repo-specific sections across the fleet sit anywhere without a finding - the model constrains the sections - it declares and nothing else. A retired name still resolves to its canonical ordinal, so a repo that has - not renamed yet is checked on order too rather than silently losing the check along with the name. - """ - findings = [] - by_name = {s["name"].lower(): s for s in model["sections"]} - retired = {r.lower(): s for s in model["sections"] for r in s.get("retiredNames", [])} - heads = ordered_headings(text, 2) - lower = [h.lower() for h in heads] - present = set(lower) - - for h, hl in zip(heads, lower): - if hl in retired: - findings.append( - ( - "LETTER", - f"readme: section '{h}' uses a retired name - rename it to '{retired[hl]['name']}', which has no accepted aliases (spec/readme-structure.md)", - ) - ) - - for s in model["sections"]: - req = s["required"] - if req == "optional" or (req == "public" and not public): - continue - if {t.lower() for t in s.get("notApplicableTo", [])} & {t.lower() for t in sel}: - continue - if s["name"].lower() in present or any( - r.lower() in present for r in s.get("retiredNames", []) - ): - continue - why = " (a public repo; optional while private)" if req == "public" else "" - findings.append( - ( - "LETTER", - f"readme: no `## {s['name']}` section{why} - it is a required section (spec/readme-structure.md)", - ) - ) - - seq = [ - (by_name.get(hl) or retired.get(hl), h) - for h, hl in zip(heads, lower) - if hl in by_name or hl in retired - ] - for (prev, prev_h), (cur, cur_h) in itertools.pairwise(seq): - if cur["ordinal"] < prev["ordinal"]: - findings.append( - ( - "LETTER", - f"readme: section '{cur_h}' follows '{prev_h}' but is ordered before it - the declared sections keep their relative order (spec/readme-structure.md)", - ) - ) - - last = next((s for s in model["sections"] if s.get("last")), None) - if last and last["name"].lower() in present and lower[-1] != last["name"].lower(): - findings.append( - ( - "LETTER", - f"readme: `## {last['name']}` is not the last section, '{heads[-1]}' follows it - it closes the file, immediately before the link definitions (spec/readme-structure.md)", - ) - ) - - for s in model["sections"]: - if s["name"].lower() not in present or not s.get("subsections"): - continue - have = {h.lower() for h in ordered_headings(readme_region(text, s["name"]) or "", 3)} - for sub in s["subsections"]: - if sub.lower() not in have: - findings.append( - ( - "LETTER", - f"readme: `## {s['name']}` carries no `### {sub}` sub-section (spec/readme-structure.md)", - ) - ) - return findings - - -def distribution_prefixes(model, slug): - """The URL prefixes that make a link this repo's own, from the model's single declaration of ownership.""" - owner = slug.split("/")[0] - return tuple( - p.replace("{slug}", slug).replace("{owner}", owner).lower() - for p in model.get("distribution", {}).get("urlPrefixes", []) - ) - - -def link_kind(url, slug, is_rendered=False, prefixes=()): - """Which linkGroups kind a reference definition points at, for the repo at `slug`. - - `is_rendered` says whether this one reference is used as an image, which makes it a shield whatever host - serves it. It is per reference rather than per URL deliberately: two names can point at one URL, and - keying on the URL would make a plain link a shield because some other reference to the same address is - rendered, which the spec's wording ("a badge, judged by the document rendering the reference as an image") - does not say. - - Keying on the host instead read a badge from any other host as a plain URI and asked for it to be renamed - `-link`, which is a rename away from the convention. The case that found it was a retired last-build - service, which deprecatedShields now reports separately, but the rule is about how a reference is used - rather than about any one host, so there is no img.shields.io short-circuit here: across the fleet all 119 - img.shields.io definitions are rendered as images, so a host test classifies nothing usage does not and - can only contradict the rule it sits beside. - - A reference is local only when it carries no URI scheme, meaning a path inside this repo. A `mailto:`, - `ssh:` or `ftp:` target is a link rather than a file, and reading it as local would demand a bare name and - the Repo group for it. No fleet README carries one today, so this is a shape the rule has to get right - before one does rather than a defect being repaired. - - Distribution is scoped to this repo's own URLs, so a link to somebody else's GitHub repo or Docker Hub - image, which a 3rd Party Tools list is full of, stays external rather than being read as a channel of this - project's. Those prefixes come from the model rather than from a second list here, since ownership stated - in two places is two things that can disagree, and a canonicalLinks entry is consulted only for a - reference this already classified distribution. - """ - if url.startswith("#"): - return "anchor" - if not url.startswith(("http://", "https://")) and not _URI_SCHEME.match(url): - return "local" - if prefixes and url.lower().startswith(tuple(prefixes)): - return "distribution" - return "shield" if is_rendered else "external" - - -def canonical_link_entry(url, model, slug): - """The canonicalLinks entry `url` is a destination for, or None where the model fixes no name for it. - - A destination every repo has is called the same thing in every repo, so a reader moving between them is - not re-learning names. - """ - base = f"https://github.com/{slug}" - bare = url.rstrip("/") - for c in model.get("canonicalLinks", []): - if "match" in c: - if re.search(c["match"], url): - return c - continue - if "repoPath" not in c: - # Loud rather than skipped, since skipping would quietly stop enforcing this destination's name. - # The schema requires one of the two and validate.py gates it, so reaching here means a model nothing checked. - raise KeyError( - f"spec/readme-sections.json: canonicalLinks entry '{c.get('name')}' carries neither repoPath nor match" - ) - want = (base + c["repoPath"]).rstrip("/") - if ( - bare.lower().startswith(want.lower()) - if c.get("prefix") - else bare.lower() == want.lower() - ): - return c - return None - - -def canonical_name_findings(defs, model, slug, rendered=(), prefixes=()): - """Reference names against canonicalLinks, resolved over the whole definition set rather than one at a time. - - A perTarget destination is one a repo may publish several of, and its rule is a count: the bare canonical - name where there is exactly one, and `-` where there are several. Judging a definition alone - cannot see which case it is in, which is why this is a second pass over the collected set. The qualifier is - a prefix on the same name rather than a different name, so one shape covers PlexCleaner's single image and - NxWitness's twelve. - - Only a distribution-kind link is renamed, so an upstream image a repo happens to link is left alone. - ESPHome-Config links `hub.docker.com/r/esphome/esphome`, which is upstream's image and not a channel of - its own, and matching the host alone told it to call that `docker-hub-link`. - """ - findings, hits = [], {} - for ref, url in defs: - if link_kind(url, slug, ref in rendered, prefixes) != "distribution": - continue - c = canonical_link_entry(url, model, slug) - if c: - hits.setdefault(c["name"], (c, []))[1].append((ref, url)) - for name, (c, group) in hits.items(): - if c.get("perTarget") and len(group) > 1: - for ref, url in group: - if not (ref.endswith(f"-{name}") and len(ref) > len(name) + 1): - findings.append( - ( - "LETTER", - f"readme: the reference `[{ref}]` points at {url} - this repo publishes {len(group)} of these, so each is named `-{name}` (spec/readme-structure.md)", - ) - ) - else: - for ref, url in group: - if ref != name: - findings.append( - ( - "LETTER", - f"readme: the reference `[{ref}]` points at {url} and is named `{name}` in every repo - a shared destination carries a shared name (spec/readme-structure.md)", - ) - ) - return findings - - -def readme_link_findings(text, model, slug): - """Reference-definition naming and grouping, per the linkGroups, linkNaming and canonicalLinks model. - - The naming half is a LETTER because the fleet already meets it: 119 of 122 shield references end - `-shield` and 514 of 532 URI references end `-link`, so the rule is written down rather than imposed. - The grouping half is a DRIFT because it is not met: the fleet carries seventeen distinct group-header - names across twenty-two repos, and gating that would bury the naming findings under a re-grouping sweep - of every README at once. - """ - findings = [] - suffixes = {n["kind"]: n["suffix"] for n in model["linkNaming"]} - prefixes = distribution_prefixes(model, slug) - groups = model["linkGroups"] - declared = [g["name"] for g in groups] - holds = {g["name"].lower(): g["holds"] for g in groups} - - # Walk the definitions in order, tracking which group header each one falls under. - # A comment counts as a group header only where a definition actually falls under it, since the closed set governs the reference-definition block and not every comment in the file. - # A tool directive is skipped outright rather than relying on that, because one placed inside the reference block does have definitions under it and would be reported as an unknown group. - # The prefixes are declared in the model rather than guessed from the shape of the text, since which tools a repo instructs is a fact about the fleet rather than something to infer. - unfenced = unfenced_text(text) - seen_headers, current, in_group = [], None, {} - for ln in unfenced.split("\n"): - s = ln.strip() - h = re.fullmatch(r"", s) - if h and "omit from toc" not in h.group(1): - if not any( - h.group(1).lower().startswith(d.lower()) for d in model.get("directiveComments", []) - ): - current = h.group(1) - continue - d = _LINK_DEF.match(s) - if d: - if current is not None and current not in in_group: - seen_headers.append(current) - in_group.setdefault(current, []).append((d.group(1), d.group(2))) - - all_defs = [p for v in in_group.values() for p in v] - # A reference the document renders as an image is a shield whatever host serves it, so resolve those first. - # Read over the unfenced text for the same reason the definitions are. - # An `![alt][ref]` inside a code sample is markup being shown rather than a badge being rendered, and counting it would make a plain link a shield. - by_ref = dict(all_defs) - rendered = {m.group(1) for m in _MD_IMAGE_REF.finditer(unfenced) if m.group(1) in by_ref} - described = {"shield": "a badge", "anchor": "an in-page anchor", "local": "a path in this repo"} - for ref, url in all_defs: - kind = link_kind(url, slug, ref in rendered, prefixes) - want = suffixes.get(kind, "") - got = "-shield" if ref.endswith("-shield") else "-link" if ref.endswith("-link") else "" - if got != want: - shown = f"`{want}`" if want else "a bare name with no suffix" - findings.append( - ( - "LETTER", - f"readme: the reference `[{ref}]` points at {described.get(kind, 'a URI')} and should end in {shown} (spec/readme-structure.md)", - ) - ) - findings += canonical_name_findings(all_defs, model, slug, rendered, prefixes) - - unknown = [h for h in seen_headers if h not in declared] - if unknown: - findings.append( - ( - "DRIFT", - f"readme: link-group header(s) outside the declared set: {', '.join(sorted(set(unknown)))} - the groups are {', '.join(declared)} (spec/readme-structure.md)", - ) - ) - ordered = [h for h in seen_headers if h in declared] - if ordered != sorted(ordered, key=declared.index): - findings.append( - ( - "DRIFT", - f"readme: the link groups run {', '.join(ordered)} rather than the declared order (spec/readme-structure.md)", - ) - ) - if in_group.get(None): - # Distinguish no grouping at all from a stray definition above the first header. - # Two repos carry the whole block ungrouped, and "116 definitions sit above the first header" describes that badly. - if not seen_headers: - findings.append( - ( - "DRIFT", - f"readme: the {len(in_group[None])} reference definitions carry no group headers - they are grouped under {', '.join(declared)} (spec/readme-structure.md)", - ) - ) - else: - findings.append( - ( - "DRIFT", - f"readme: {len(in_group[None])} reference definition(s) sit above the first group header (spec/readme-structure.md)", - ) - ) - for header, defs in in_group.items(): - if header is None or not defs: - continue - names = [r for r, _ in defs] - if names != sorted(names): - findings.append( - ( - "DRIFT", - f"readme: the `{header}` group is not sorted by reference name (spec/readme-structure.md)", - ) - ) - want_kind = holds.get(header.lower()) - strays = ( - {link_kind(u, slug, r in rendered, prefixes) for r, u in defs} - {want_kind} - if want_kind - else set() - ) - if strays: - findings.append( - ( - "DRIFT", - f"readme: the `{header}` group holds {', '.join(sorted(strays))} reference(s) where it holds {want_kind} (spec/readme-structure.md)", - ) - ) - return findings - - -# A tool row is read as text for its reference name, with both outer pipes optional as GitHub's Markdown makes them. -_TOOL_ROW = re.compile(r"^\|?\s*\[([^\]]+)\]\[([^\]]+)\]\s*\|\s*([^|]*?)\s*(?:\||$)") -_TOOL_BULLET = re.compile(r"^[-*]\s*\[([^\]]+)\]\[([^\]]+)\]\s*(.*)$") - - -def third_party_tool_findings(text, catalog): - """A repo's 3rd Party Tools entries against the shared catalog in spec/third-party-tools.json. - - The catalog is a standard set rather than a complete one. A repo's tools are mostly its own, so a tool the - catalog does not name produces nothing at all: what is checked is the intersection, meaning that a repo - using a tool the fleet has standardized links it by the same URL and describes it the same way. Twelve - tools already appear in more than one repo, and three of them are linked by two different URLs today, which - is the divergence this closes. - - Matching is by the tool's display name, the text a README links, since that is what a reader compares - across repos and what the catalog is keyed on. - - The License column is read off the table's header rather than off its rows, so it reports once for the - table instead of once per tool, and it is the only extra column named. spec/readme-structure.md forbids - that one column and no other, and a check that flagged every extra column would be grading the fleet on a - rule nobody wrote. - """ - body = readme_region(text, "3rd Party Tools") - if body is None: - return [] # the absent section is already one finding from readme_section_findings - declared = {t["name"].lower(): t for t in catalog["tools"]} - # Read over the unfenced text, as readme_link_findings does: a sample footnote is not a definition. - defs = {m.group(1): m.group(2) for m in _LINK_DEF.finditer(unfenced_text(text))} - findings = [] - listed = [] - in_head = False - for tok in _MD.parse(body): - if tok.type in ("thead_open", "thead_close"): - in_head = tok.type == "thead_open" - elif in_head and tok.type == "inline" and tok.content.strip().lower() == "license": - findings.append( - ( - "LETTER", - "readme: 3rd Party Tools carries a License column - a license belongs to the dependency and is authoritative at its source, so restating it adds a maintenance obligation and no information (spec/readme-structure.md)", - ) - ) - for ln in body.split("\n"): - m = _TOOL_ROW.match(ln.strip()) or _TOOL_BULLET.match(ln.strip()) - if not m: - continue - name, ref, desc = m.group(1), m.group(2), m.group(3).strip(" -:") - listed.append(name) - want = declared.get(name.lower()) - if want is None: - continue - url = defs.get(ref) - if url is not None and url.rstrip("/") != want["link"].rstrip("/"): - findings.append( - ( - "LETTER", - f"readme: 3rd Party Tools links {name} as {url} where the fleet links it as {want['link']} - a shared tool carries one link (spec/third-party-tools.json)", - ) - ) - if desc != want["description"]: - shown = f"'{desc}'" if desc else "no description" - findings.append( - ( - "LETTER", - f"readme: 3rd Party Tools describes {name} as {shown} where the fleet describes it as '{want['description']}' - a shared tool carries one description (spec/third-party-tools.json)", - ) - ) - # Ordering covers every tool listed, not only the cataloged ones, since a reader scans the whole list. - if listed != sorted(listed, key=str.lower): - first = next( - f"{a} before {b}" for a, b in itertools.pairwise(listed) if a.lower() > b.lower() - ) - findings.append( - ( - "LETTER", - f"readme: 3rd Party Tools is not alphabetized ({first}) - the list is scanned rather than read (spec/readme-structure.md)", - ) - ) - return findings - - -def readme_shield_findings(text, model, entry): - """Shield presence, by the additive classes in spec/readme-sections.json. - - Each shield names the section it belongs in, so the license shield is an ordinary member of the base class - that happens to sit in the closing License section rather than a second model beside this one. Where a - named section is absent, the shield is skipped, because readme_section_findings already reports the - section and a second finding would describe the same gap. - - A class is a floor, so an extra shield is never a finding. The classes deliberately assert the endpoint - and not the channel: a repo publishing several images carries a version shield per image per channel - (NxWitness carries forty, across six images and four channels that are not `develop`), so a - latest-plus-develop pair is the single-image form the spec recommends rather than something every docker - repo can be measured against. - """ - findings = [] - # One unfenced view serves both passes, so the definitions and the rendered images cannot be read from different documents. - unfenced = unfenced_text(text) - defs = {m.group(1): m.group(2) for m in _LINK_DEF.finditer(unfenced)} - # A retired badge service is scanned across the whole document rather than per section, since a dead badge is wrong wherever it sits. - # It renders broken rather than absent, which a visitor reads as a failing build rather than as a stale badge. - # Both forms are read, since reading definitions alone made an inline badge invisible rather than wrong, which is the reading shield_endpoints already takes for every other shield. - # A definition is reported even where nothing renders it, because a retired service left in the reference block is removed with the badge rather than after it. - # Which of the four it is decides the wording, since a definition nothing renders is not rendering anything and saying so sends the reader looking for a badge that is not on the page. - # Attribution is by reference name and never by URL: the same endpoint rendered inline leaves this definition unused, so reading the URL alone would credit a render to a reference nothing uses. - # A set rather than the list shield_endpoints returns, since every use here is membership or a difference, and the names say which of the two namespaces each holds. - rendered_urls = set(shield_endpoints(unfenced, defs)) - used_refs = {m.group(1) for m in _MD_IMAGE_REF.finditer(unfenced)} - for dep in model.get("deprecatedShields", []): - defined_urls = set() - for ref, url in sorted(defs.items()): - if dep["match"] in url: - defined_urls.add(url) - if ref in used_refs: - verb = f"renders {dep['label']}" - elif url in rendered_urls: - verb = f"defines {dep['label']} and it is rendered elsewhere" - else: - verb = f"defines {dep['label']} and nothing renders it" - findings.append( - ( - "LETTER", - f"readme: `[{ref}]` {verb}, which is retired - {dep['reason']} (spec/readme-structure.md)", - ) - ) - for url in sorted({u for u in rendered_urls if dep["match"] in u} - defined_urls): - findings.append( - ( - "LETTER", - f"readme: an inline image renders {dep['label']}, which is retired - {dep['reason']} (spec/readme-structure.md)", - ) - ) - targets = {(p.get("target") if isinstance(p, dict) else p) for p in entry.get("publish", [])} - secrets = set(entry.get("requiredSecrets", [])) - want = [] - for cls in model["shieldClasses"]: - t = cls["trigger"] - if ( - t["kind"] == "always" - or (t["kind"] == "publish" and t.get("target") in targets) - or (t["kind"] == "secret" and t.get("name") in secrets) - ): - want += cls["shields"] - for sh in want: - region = addressed_region(text, sh["in"]) - if region is None: - continue - if not any(shield_matches(u, sh) for u in shield_endpoints(region, defs)): - queries = "".join(f" with `{q}`" for q in sh.get("requireQuery", [])) - if sh.get("forbidQuery") is not None: - queries += f" without `{sh['forbidQuery']}`" - findings.append( - ( - "LETTER", - f"readme: `{sh['in']}` carries no {sh['label']} shield ({sh['match']}{queries}), which this repo's deliverables require (spec/readme-structure.md)", - ) - ) - if not sh.get("exclusive"): - continue - # Rendered elsewhere, which for the license shield is the whole point: it belongs at the bottom and nowhere else. - # Compared over uses rather than over raw text, so the shield's own `[ref]: url` definition is not read as a second placement. - outside = strip_sections( - _HTML_COMMENT.sub("", unfenced_text(text)), [sh["in"].split(">")[0].strip()] - ) - if any(shield_matches(u, sh) for u in shield_endpoints(outside, defs)): - findings.append( - ( - "LETTER", - f"readme: the {sh['label']} shield is rendered outside `{sh['in']}` - it belongs there and nowhere else (spec/readme-structure.md)", - ) - ) - return findings - - -def description_findings(doc_texts, entry, live, slug): - """The README title/tagline, the GitHub About description, and the Docker Hub short description, checked as one mirror set. - - Per spec/readme-structure.md item 1 and GOVERNANCE.md "Repository Details", the H1 is the repo name. The tagline - after it, the first line of the intro region, is a link-free plain sentence of at most 100 characters that - carries verbatim to the GitHub About description, and on a docker repo to the Docker Hub short description. Any - further paragraph is free prose no mirror reads, which is why only the first line is measured. - - registry/repos.json's optional `description` is the canonical value once a repo declares it (GOVERNANCE.md - "Repository Details"). repo-config/configure.sh then writes the About panel from that same field. A repo that - has not adopted it yet still has the README as its source of truth, so the checks below fall back to the - tagline exactly as before. - - A declared field's mirrors (About, Docker Hub) are measured against it even where the README itself could - not be read (missing, oversized, or fetched with no inline content) or carries no usable tagline, since the - declared value is canonical on its own and does not need the README to establish it. - """ - findings = [] - declared = None - if "description" in entry: - # Delegates to validate.py's own contract instead of re-checking a second, easily-incomplete copy of it - # (an earlier version here missed the link and length rules, accepting either as canonical). - shape_errors = validate.description_errors(slug, entry["description"]) - if shape_errors: - findings += [ - ( - "DEFECT", - f"{msg}. Treated as undeclared here, since it fails spec/validate.py's contract.", - ) - for msg in shape_errors - ] - else: - declared = entry["description"] - readme_want = None - if "README.md" in doc_texts: - title, intro = title_and_intro(doc_texts["README.md"]) - intro_line = tagline(intro) - refs = references(doc_texts["README.md"]) - # The H1 is the repository name, and a hyphenated name may render its hyphens as spaces. - # Use the GitHub API's canonical name, since the registry-URL slug can carry a different case. - repo_name = live.get("name") or slug.split("/")[-1] - if not title: - findings.append( - ( - "LETTER", - "readme: no `# ` H1 title - the README opens with `# ` then the tagline (spec/readme-structure.md)", - ) - ) - elif title.replace("-", " ") != repo_name.replace("-", " "): - findings.append( - ( - "LETTER", - f"readme: the H1 title '{title}' is not the repo name '{repo_name}' (a hyphenated name may render its hyphens as spaces) - the H1 is the repository name (spec/readme-structure.md)", - ) - ) - if not intro_line: - findings.append( - ( - "LETTER", - "readme: no tagline after the H1 - the README opens with the title then a one-line description, which doubles as the About description (spec/readme-structure.md)", - ) - ) - else: - if strip_md_links(intro_line, refs) != intro_line: - findings.append( - ( - "LETTER", - "readme: the tagline carries Markdown links - keep it link-free plain text, it doubles as the repo About description (spec/readme-structure.md)", - ) - ) - readme_want = strip_md_links(intro_line, refs).strip() - if len(readme_want) > 100: - findings.append( - ( - "LETTER", - f"readme: the tagline is {len(readme_want)} characters, over the 100-char limit (Docker Hub's short-description cap, the tightest surface it feeds) - tighten it to one short sentence (spec/readme-structure.md)", - ) - ) - # The declared field wins once a repo has one; every mirror (README, About, Docker Hub) is then measured against it. - # A repo with no declared field keeps the README as the source, exactly as before, so with no readable tagline - # there is nothing yet to measure the other mirrors against. - want = declared or readme_want - if want is None: - return findings - source = "registry/repos.json" if declared else "the README" - if declared and readme_want is not None and readme_want != declared: - findings.append( - ( - "LETTER", - f"readme: the tagline ('{readme_want}') does not match the declared description ('{declared}') - registry/repos.json's description is canonical once declared, so the README follows it (GOVERNANCE.md Repository Details)", - ) - ) - desc = (live.get("description") or "").strip() - if desc != want: - findings.append( - ( - "LETTER", - f"description: the About description does not match the {'declared description' if declared else 'README tagline'} (description '{desc}' vs '{want}') - set it from {source}, or sharpen {source} first if the description carries real detail (GOVERNANCE.md Repository Details)", - ) - ) - # Docker Hub short description mirrors the same canonical value, for a repo that publishes a docker image. - # A transient lookup failure surfaces as a DRIFT ("could not verify"), never aborting or silently passing. - # A 404 (image not at the derived name) returns None and is skipped. - if any( - (pt.get("target") if isinstance(pt, dict) else pt) == "docker" - for pt in entry.get("publish", []) - ): - try: - dh = docker_hub_description(slug) - except Exception as e: # noqa: BLE001 - dh = None - findings.append( - ( - "DRIFT", - f"description: could not read the Docker Hub short description to verify it mirrors {source} ({e}) - verify by hand", - ) - ) - if dh is not None and dh.strip() != want: - findings.append( - ( - "LETTER", - f"description: the Docker Hub short description ('{dh.strip()}') does not match '{want}' - set it from {source} (spec/readme-structure.md)", - ) - ) - return findings - - def workspace_cspell_words(text): - """True if workspace/settings JSON carries its own cSpell word list - the block cspell.json canonicalizes. - - Matches the quoted setting keys case-insensitively, so a mere mention of the cspell.json file is not a hit. - """ + """True if workspace/settings JSON carries its own cSpell word list.""" low = text.lower() return any( key in low for key in ('"cspell.words"', '"cspell.userwords"', '"cspell.ignorewords"') @@ -1902,11 +854,7 @@ def job_nodes(text): def code_view(node, depth=0): - """A node's keys and scalar values as lines indented two spaces per level, the shape a contract token is read in. - - A job's children sit at `depth` 2, its place in a workflow. A block-scalar body is a script or prose rather - than structure, so it is left out, except an `if:` condition, folded onto one line. - """ + """A node's keys and scalar values as lines indented per level, the shape a contract token is read in.""" out = [] def walk(child, key, level): @@ -1932,10 +880,7 @@ def walk(child, key, level): def split_jobs(text): - """Each job's own source lines, `{key: text}`, cut at the parser's marks for the verbatim job hash. - - A comment dedented below the job keys belongs to no job, so it is left out of every region. - """ + """Each job's own source lines, `{key: text}`, cut at the parser's marks for the verbatim job hash.""" jobs = _mapping(_compose(text)).get("jobs") if not isinstance(jobs, yaml.MappingNode) or jobs.flow_style or not jobs.value: return {} @@ -2015,11 +960,7 @@ def normalize_declared_python_directory(line): def python_directories_caller_findings(path, text, entry): - """DRIFT where a caller's declared `python-directories` input disagrees with the registry's own. - - Compared as normalized sets of declarations, since naming `.` is not the same as naming nothing. - Only the job keyed `validate` is compared, and validate-task.yml called from another job is reported. - """ + """DRIFT where a caller's declared `python-directories` input disagrees with the registry's own.""" if path not in PYTHON_DIRECTORIES_CALLERS: return [] jobs = job_nodes(text) @@ -2098,10 +1039,7 @@ def _scalar_text(value): def _schedule_never_publishes(doc): - """True where `on:` admits a `schedule` and no `push`, and every build-release-task.yml job requires a dispatch. - - The dispatch test must be a top-level conjunct of the job's `if:`, in a condition holding no `||` or negated group. - """ + """True where `on:` admits a `schedule` and no `push`, and every build-release-task.yml job requires a dispatch.""" # PyYAML reads a bare `on` key as the boolean true, per YAML 1.1. on = doc.get("on", doc.get(True)) triggers = set(on) if isinstance(on, (dict, list)) else {on} @@ -2122,12 +1060,7 @@ def _schedule_never_publishes(doc): def concurrency_caller_findings(path, text): - """DRIFT where the publisher or the merge-bot declares a concurrency group other than the one WORKFLOW.md pins. - - The publisher's group is the ref-scoped base plus at most the push-only run-id suffix, or the schedule-only - suffix where its schedule never publishes. The merge-bot's is the base keyed on the PR number. Either leaves - `cancel-in-progress` false or unset. - """ + """DRIFT where the publisher or the merge-bot declares a concurrency group other than the one WORKFLOW.md pins.""" base = CONCURRENCY_CALLERS.get(path) if base is None: return [] @@ -2215,11 +1148,7 @@ def content_hash(text): def classify_verbatim(down_text, canon_text, past_texts): - """None if the downstream copy matches the current canonical, 'stale' if it matches a past hub revision - (the base advanced - re-vendor), or 'modified' if it matches no revision the base ever produced (the - repo changed fixed content). The discriminator is a content hash, never a version stamp - a stamp can - claim to be current while the body was edited, so it is never trusted for integrity. - """ + """None if the copy matches the current canonical, 'stale' if it matches a past hub revision, else 'modified'.""" dh = content_hash(down_text) if dh == content_hash(canon_text): return None @@ -2235,11 +1164,7 @@ class ShallowCloneError(RuntimeError): @functools.cache def _require_full_history(root): - """Raise when `root` is a shallow clone, since every history read there is silently wrong. - - The probe is the git-path `shallow` file, which every git version writes for a shallow clone. - Git before 2.15 echoes an unknown `--is-shallow-repository` flag, so that flag proves nothing. - """ + """Raise when `root` is a shallow clone, since every history read there is silently wrong.""" probe = subprocess.run( ["git", "rev-parse", "--git-path", "shallow"], capture_output=True, @@ -2260,16 +1185,7 @@ def _require_full_history(root): @functools.cache def _hub_main_rev(): - """The hub's own `main`, fetched fresh from `origin` and resolved to a commit SHA. - - Reached as a checkout of one's own, fetched immediately before use, per AGENTS.md, rather - than trusting ROOT's checked-out branch. `git fetch` - writes into ROOT's own object database, so a separate clone is not needed. Resolved to a - concrete SHA right away, not the mutable `FETCH_HEAD` pointer, so a later fetch elsewhere in - the process cannot move it mid-run. - - Cached: one fetch and resolve per run, reused for every rel_path read. - """ + """The hub's own `main`, fetched fresh from `origin` and resolved to a commit SHA.""" fetch = subprocess.run( ["git", "fetch", "-q", "origin", "main"], cwd=ROOT, @@ -2297,21 +1213,7 @@ def _hub_main_rev(): def _git_revisions(rel_path, rev=None): """Every commit that touched rel_path in the hub's history, newest first, as (date, sha, text). - `text` is None where rel_path has no file content at this revision: absent (deleted, checked - against the revision's own tree rather than `git show`'s stderr wording, which reads - differently once rel_path exists again in a later commit), or present as something other than - a regular file, by ls-tree mode rather than type (a directory from a file-to-directory - transition, a symlink, whose ls-tree type is "blob" too but whose content is its target path - rather than a file's, a submodule gitlink). A `git show` failure for any other reason (a - permission or encoding fluke, a corrupt object) raises instead of folding into the same None, - so a real command fault cannot pass as an ordinary absence. Cached because one canonical's - history is read once per fidelity/staleness check, then reused for every audited repo's copy. - - `rev` names the git revision walked. It defaults to the hub's own `main` via - `_hub_main_rev()` rather than the implicit HEAD of whatever - branch ROOT has checked out. The --selftest fixtures pass an explicit `rev` to exercise a - plain local branch in a throwaway repo with no `origin` to fetch, keeping the offline engine - self-test offline. + `text` is None where rel_path is absent or not a regular file at that revision, and any other git failure raises. """ _require_full_history(ROOT) walk_rev = _hub_main_rev() if rev is None else rev @@ -2324,9 +1226,7 @@ def _git_revisions(rel_path, rev=None): check=False, ) if r.returncode != 0: - # A real command failure (not a git repo, a corrupt object) must not read as "no - # history": that silently clears the intent-staleness advisory and drops verbatim's - # past-revision list, both misreporting a tool fault as a clean audit. + # A real command failure must not read as no history. raise RuntimeError(f"git log failed for {rel_path}: {r.stderr.strip()}") if not r.stdout.strip(): return [] @@ -2343,21 +1243,13 @@ def _git_revisions(rel_path, rev=None): check=False, ) if t.returncode != 0: - # A real lookup failure (a corrupt object, not this sha): git ls-tree exits non-zero - # only for that, never for a merely absent path (empty stdout, exit 0, below). raise RuntimeError(f"git ls-tree failed for {sha}:{rel_path}: {t.stderr.strip()}") entry = t.stdout.strip() mode = entry.split(None, 1)[0] if entry else None if mode not in ("100644", "100755"): - # Absent (empty stdout), or present as something other than a regular file (a - # directory from a file-to-directory transition, a symlink, a submodule gitlink): - # none has file content to compare, so all read the same as a confirmed deletion. A - # symlink's ls-tree type is "blob" too (its content is the link target), so the mode - # is checked directly rather than the type. out.append((date, sha, None)) continue # Decode as UTF-8 with replacement to match the downstream and canonical reads. - # A divergent decode would fabricate a mismatch. s = subprocess.run( ["git", "show", f"{sha}:{rel_path}"], cwd=ROOT, @@ -2378,36 +1270,19 @@ def git_file_history(rel_path): def canonical_current_text(rel_path): - """The hub's current canonical content of rel_path, from the same `_git_revisions()` call - `git_file_history()` and `hub_last_change()` already make, so "current" and "history" can - never disagree about which commit they read: reading "current" from ROOT's working tree - while history walked the resolved `main` SHA let a copy that matches today's main misclassify - as stale against its own most recent history entry. - None if rel_path has no history at `main`, or its newest revision has no file content there. + """The hub's current canonical content of rel_path, from the same history read the stale check uses. + + None if rel_path has no history at `main`, or its newest revision has no file content. """ revisions = _git_revisions(rel_path) return revisions[0][2] if revisions else None def _last_effective_change(revisions): - """The (date, sha) of the newest revision in `revisions` (newest-first (date, sha, text) - triples for one file) whose content differs from its predecessor after normalize(), or the - oldest (creation) revision if every later one differs from its predecessor only by normalized - churn (spec/fidelity-model.md "Normalization": EOL, a Dependabot action-pin bump, a pruned - `needs:` list). None if `revisions` is empty. - - The creation revision always counts as effective: it has no predecessor, and normalize() - applied to only one side proves nothing. A revision with unreadable text (None) also counts as - effective rather than being silently skipped, since a real difference cannot be ruled out. - - `revisions` is assumed newest-first with each entry the immediate predecessor of the one - before it (verified empirically over every intent-fidelity canonical file's full history): - this repo's own branching is forward-only (no back-merges from - main into develop) with feature branches squash-merged one at a time, so a canonical file's - per-path `git log` is a single line, not a graph with siblings to mis-order. A canonical file - reached through a genuinely branching history (a direct hotfix to main alongside independent - develop work touching the same path) would need each revision compared against its actual git - parent(s) instead of its list neighbor. + """The (date, sha) of the newest revision whose content differs from its predecessor after normalize(). + + `revisions` is newest-first (date, sha, text) triples, and an empty list returns None. + The oldest revision answers where every later one is normalized churn, and a None text counts as a change. """ for i, (date, sha, text) in enumerate(revisions): if i + 1 == len(revisions): @@ -2420,13 +1295,7 @@ def _last_effective_change(revisions): @functools.cache def git_blob_in_file_history(rel_path, blob_sha, rev=None): - """Whether a blob occurred in a path's hub history. - - `rev` defaults to the hub's own `main` via `_hub_main_rev()` - for the same reason `_git_revisions` does: ROOT's checked-out branch is not necessarily - `main`, and a develop-only revision matching `blob_sha` must not read as "stale" against a - hub history `main` doesn't actually contain. - """ + """Whether a blob occurred in a path's hub history.""" _require_full_history(ROOT) walk_rev = _hub_main_rev() if rev is None else rev result = subprocess.run( @@ -2442,17 +1311,7 @@ def git_blob_in_file_history(rel_path, blob_sha, rev=None): @functools.cache def hub_last_change(rel_path): - """The hub checkout's most recent EFFECTIVE change to rel_path, as (iso_date, short_sha), or - None if untracked. - - "Effective" excludes normalized churn (spec/fidelity-model.md "Normalization": EOL, a - Dependabot action-pin bump, a pruned `needs:` list) the same way the verbatim check already - does, via _last_effective_change over the file's full history. A raw last-commit date treated - every one of those bumps as the copy "trailing", even though the fidelity model already - classifies that class as governed per-repo drift rather than a deviation. - - Cached because one canonical's date is compared against every audited repo's copy. - """ + """The hub checkout's most recent EFFECTIVE change to rel_path, as (iso_date, short_sha), or None if untracked.""" change = _last_effective_change(_git_revisions(rel_path)) if change is None: return None @@ -2461,14 +1320,7 @@ def hub_last_change(rel_path): def intent_canonical_rel(item, path): - """The hub path an intent unit's copy is judged against for staleness. - - `reference` wins where the manifest sets one. - Otherwise the intent unit's own canonical, `intentRef`, wins. - Otherwise the unit compares against its own `path`. - Only `intentRef` ever carries a `#anchor`, routing a reader to one section of a larger doc. - The anchor names a place to read, not a narrower file to diff against, so it is stripped there and nowhere else, comparing the whole canonical file instead. - """ + """The hub path an intent unit's copy is judged against for staleness.""" # spec/validate.py shape-checks these fields, but this engine runs standalone and does not invoke it first. ref = item.get("reference") if isinstance(ref, str) and ref: @@ -2482,10 +1334,7 @@ def intent_canonical_rel(item, path): def declared_alternatives(baseline): - """Map each baseline path to the `alternatives` its entry declares, the other paths it may sit at. - - spec/validate.py holds a path declaring alternatives to a single entry, so nothing here merges. - """ + """Map each baseline path to the `alternatives` its entry declares, the other paths it may sit at.""" return { item["path"]: list(item["alternatives"]) for item in baseline if item.get("alternatives") } @@ -2494,8 +1343,7 @@ def declared_alternatives(baseline): def contents_text(content): """The decoded text of a contents API response, or None where it holds no inline file content. - A directory returns a list, and a too-large file returns encoding "none". An empty file decodes - to an empty string, so the guard is on the encoding rather than on truthiness. + An empty file decodes to an empty string. """ if not isinstance(content, dict) or content.get("encoding") != "base64": return None @@ -2503,11 +1351,7 @@ def contents_text(content): def read_baseline_file(slug, ground, path, alternatives=()): - """The path a baseline file was found at and its contents response, or `(path, None)`. - - The declared path is read first and an alternative only while nothing is found. A repo holding - both is therefore read at the declared path. - """ + """The path a baseline file was found at and its contents response, or `(path, None)`.""" for candidate in (path, *alternatives): content = gh(f"repos/{slug}/contents/{candidate}?ref={ground}", ok404=True) if content is not None: @@ -2518,11 +1362,7 @@ def read_baseline_file(slug, ground, path, alternatives=()): def baseline_file_findings(slug, ground, path, alternatives, item, is_hub): """Read one baseline file, returning `(content, text, findings)` for what reading it alone owes. - Where no path is present, an interface unit's absence is DRIFT, since a workflow's naming varies - more than a carried config's. Any other unit's absence is a file-presence LETTER. `text` is None - where the contents API returned no inline content, as for a too-large file, while an empty file - decodes to an empty string. An intent unit outside the hub gets its staleness advisory, read at - the path the file was found at. The hub's own copies are the canonicals, so it has none. + `text` is None where the API returned no inline content, and an empty file is an empty string. """ found, content = read_baseline_file(slug, ground, path, alternatives) fid = item.get("fidelity") if item else "presence" @@ -2555,21 +1395,7 @@ def baseline_file_findings(slug, ground, path, alternatives, item, is_hub): def check_intent_staleness(slug, ground, path, canonical_rel, down_text): - """The intent-staleness advisory: a last-modified comparison, since intent has no content check. - - An intent unit is judged by meaning, so the audit asserts presence and nothing about content - (spec/fidelity-model.md), which is how a copy trailed the hub by many revisions while every - check read clean. No reconciliation record exists anywhere, so the implementable proxy is - when each side last changed: the hub canonical changing after the repo's copy marks the copy - as possibly trailing. "Changed" excludes normalized churn (hub_last_change), so a Dependabot - action-pin bump or a needs-list prune does not by itself mark every carrier as trailing. - Advisory only, DRIFT and never a failure, and honest about its blind spot: a copy touched - after the hub change without actually reconciling reads current. - - A copy content-identical to the canonical cannot trail it, so that case is skipped however - old the copy's last commit is. It is also the promotion candidate spec/fidelity_honesty.py - exists to find, where the structural fix is verbatim fidelity rather than a better advisory. - """ + """The intent-staleness advisory: a last-modified comparison, since intent has no content check.""" hub_change = hub_last_change(canonical_rel) if hub_change is None: return [] @@ -2597,11 +1423,8 @@ def check_intent_staleness(slug, ground, path, canonical_rel, down_text): def check_verbatim(label, down_text, canonical_rel, extract=None): - """Compare a downstream copy against the hub's canonical (a region if `extract` is given), EOL-normalized, - and classify a mismatch as stale or modified via the canonical's git history. All findings are DRIFT: a - byte diff is a hint to review, never proof of breakage. - """ - # canonical_current_text() and git_file_history() both read the same _git_revisions() call, so a copy matching today's main can never mismatch against its own history's newest entry. + """Compare a copy, or the region `extract` returns, against the hub canonical, classifying a mismatch.""" + # Current text and history read the same _git_revisions() call, so a copy matching main cannot mismatch its own history. canon_text = canonical_current_text(canonical_rel) if canon_text is None: return [ @@ -2638,11 +1461,7 @@ def check_verbatim(label, down_text, canonical_rel, extract=None): def classify_branch_drift(base, main, develop): - """Split the paths main changed since the merge-base by direction, given {path: object-sha} maps - for the merge-base, main-head, and develop-head trees. `behind`: main moved the path and develop - still holds the merge-base object - a genuine forward-sync gap. `diverged`: both branches moved - the path, so develop may already supersede main and a blind sync from main would revert it. A - path only develop moved is not main-side drift and is excluded. Returns (behind, diverged).""" + """`(behind, diverged)` paths main changed since the merge-base, from {path: object-sha} maps of the three trees.""" changed_on_main = {p for p in set(base) | set(main) if base.get(p) != main.get(p)} differ = [p for p in changed_on_main if main.get(p) != develop.get(p)] behind = sorted(p for p in differ if develop.get(p) == base.get(p)) @@ -2653,14 +1472,7 @@ def classify_branch_drift(base, main, develop): def ground_branch_of(entry, branch=None, defaults=None): """The branch this audit reads, the registry's `groundTruthBranch` unless overridden. - Resolved in three steps, the entry, then the registry defaults, then the fleet value, which is the shape - audit_repo() already uses for workflowModel. Every reader used to fall back to a literal "main" and consult no - defaults at all, so a registry setting `defaults.groundTruthBranch` to develop validated clean and every repo - relying on it was still audited against main, with nothing reporting that the declaration went unread. - - Presence is the test at both levels rather than truthiness, matching spec/validate.py, which holds the defaults - key to the same grammar as the per-repo one: a declared empty string is a value that gate refuses by name, not - a field to default away here. + A declared empty string is kept for spec/validate.py to refuse, rather than defaulted away. """ if branch: return branch @@ -2728,16 +1540,12 @@ def coverage_types(self): @functools.cached_property def baseline(self): - """`{path: (item, sections, verbatim sections)}` for each baseline file applying here, in manifest order. - - Sections union across same-path entries, and the last fidelity-bearing entry for a path is its item. - """ + """`{path: (item, sections, verbatim sections)}` for each baseline file applying here, in manifest order.""" out = {} for item in self.spec["files"]["baseline"]: path = item["path"] if not applies(item.get("appliesTo", "*"), self.sel): continue - # A coverage feature file is N/A where no type claims codecov (spec/type-model.md). if path == "codecov.yml" and not self.coverage_types: continue prior, needed, verbatim = out.get(path, (None, set(), set())) @@ -2753,7 +1561,7 @@ def alternatives(self): @property def docs(self): - """README.md and HISTORY.md as read, for the checks that compare the two.""" + """README.md and HISTORY.md as read, for the checks that compare them.""" return { p: self.content[p] for p in ("README.md", "HISTORY.md") @@ -2804,7 +1612,6 @@ def audit_repo(entry, spec, branch=None): if check["scope"] == "repo": run.findings += check_function(check)(run) elif i == 0 or checks[i - 1]["scope"] == "repo": - # A run of per-file checks reads the baseline file by file, so findings group by file. stage = list(itertools.takewhile(lambda c: c["scope"] != "repo", checks[i:])) for path in run.baseline: for each in stage: @@ -2824,10 +1631,7 @@ def check_branch_facts(run): def check_branch_drift(run): - """Main-side changes develop lacks, split by direction, from the merge-base tree rather than the capped compare. - - Identical head trees are the fast path, since a merge-commit promotion leaves main permanently ahead. - """ + """Main-side changes develop lacks, split by direction, from the merge-base tree rather than the capped compare.""" if run.main is None or run.develop is None: return [] if ( @@ -2950,10 +1754,7 @@ def check_secrets_stems(run): def check_secrets_names(run): - """Secret names per store, never values, against what the applicable mechanisms require and forbid. - - The codecov mechanism follows the coverage-claiming types, so a language with no tests requires no token. - """ + """Secret names per store, never values, against what the applicable mechanisms require and forbid.""" secrets = run.spec["secrets"] stores = {} # No ok404, since an empty store returns an empty list and a 404 is a failed call. @@ -2999,10 +1800,7 @@ def check_secrets_names(run): def check_dependabot_ecosystems(run): - """An ecosystem the tree implies, github-actions for workflows and devcontainers for a .devcontainer, is declared. - - Presence only. An absent dependabot.yml is already a file LETTER. - """ + """An ecosystem the tree implies, github-actions for workflows and devcontainers for a .devcontainer, is declared.""" text = contents_text( gh(f"repos/{run.slug}/contents/.github/dependabot.yml?ref={run.ground}", ok404=True) ) @@ -3088,7 +1886,7 @@ def check_verbatim_file(run, path): def _sections(run, path): - """The sections a Markdown file owes here, `(presence, verbatim)`, both empty for any other file.""" + """The sections a Markdown file owes here, `(presence, verbatim)`, empty for any other file.""" _, needed, verbatim = run.baseline[path] return (needed, verbatim) if path.endswith(".md") else (set(), set()) @@ -3369,16 +2167,13 @@ def check_registry_driftnotes(run): def _selftest(): - """Run the self-test with no variable naming a repository in the environment. - - A git hook sets `GIT_DIR`. A case building a temporary repository would otherwise build it inside the hook's own repository. - """ + """Run the self-test with no variable naming a repository in the environment.""" with git_location_cleared(): return _selftest_cases() def _selftest_cases(): - """A smoke check of each helper family on constructed fixtures; tests/test_audit_helpers.py holds the suite.""" + """A smoke check of each helper family on constructed fixtures.""" pr = "name: T\non: pull_request\njobs:\n check:\n name: Check job\n runs-on: ubuntu-latest\n" contract = {"requiredJobKeys": ["check"], "requiredCheckName": "Check job"} publisher = ( @@ -3426,12 +2221,7 @@ def _selftest_cases(): def render_issue(entry, findings, ground, audited_sha, run_utc, hub_sha): - """A ready-to-file convergence issue (title, body) generated from one repo's audit findings. - - The content is a view over the audit, not composed by hand, so it is always accurate and regenerable - - re-run the audit, re-generate the issue. Findings are grouped by what the maintainer does with them: - presence/contract findings to fix, drift to converge (re-vendor or review), and anything unverifiable. - """ + """A ready-to-file convergence issue (title, body) generated from one repo's audit findings.""" name = entry["name"] types = ", ".join(declared_types(entry)) or "untyped" stamp = f"{ground}@{audited_sha[:7]}" if audited_sha else ground @@ -3442,7 +2232,7 @@ def render_issue(entry, findings, ground, audited_sha, run_utc, hub_sha): out = [] w = out.append - # No H1 - GitHub renders the issue title as the heading, so an H1 here would duplicate it. + # No H1, since GitHub renders the issue title as the heading. w( f"Generated from the hub audit of `{name}` ({types}). Run stamp `audit run {run_utc} | hub {hub_sha}`, " f"against `@ {stamp}` (the format AUDIT.md section 8 says a derived artifact quotes). Regenerate with " @@ -3511,9 +2301,6 @@ def main(argv=None): a = parse_args(argv) if a.selftest: return _selftest() - # The override reaches the same path segment and the same `?ref=` value the registry's own groundTruthBranch does, - # so it is held to the same grammar. Validating only the declared value would leave `--branch 'main?per_page=1'` - # retargeting every read, which is the request-goes-elsewhere shape rather than a request that fails. if a.branch is not None and not validate.GROUND_TRUTH_BRANCH_RE.search(a.branch): shape = validate.GROUND_TRUTH_BRANCH_SHAPE print(f"--branch {a.branch!r} does not address unencoded ({shape})", file=sys.stderr) @@ -3529,22 +2316,17 @@ def main(argv=None): "divergences": load("spec/divergences.json"), } issue_mode = a.issue - # Read once here, since both the issue path and the sweep resolve a ground-truth branch through it. reg_defaults = spec["registry"].get("defaults", {}) wanted = {n.casefold() for n in a.names} repos = [r for r in spec["registry"]["repos"] if r.get("status") == "cataloged"] if wanted: # Keyed on the entry name rather than the owner/repo identity, and a name is unambiguous because spec/validate.py refuses one that duplicates another up to case, when it is run. - # Casefolded rather than lowercased, and identically on both sides, because the two differ on a name carrying a sharp s: lowercasing keeps the character where casefolding spells it as a double s. - # A name declared with one and typed with the other would match under one normalizer and not the other, so both sides use the one spec/validate.py dedupes on. repos = [r for r in repos if r["name"].casefold() in wanted] missing = wanted - {r["name"].casefold() for r in repos} if missing: print(f"Not cataloged: {', '.join(sorted(missing))}", file=sys.stderr) return 2 - # Findings are a point-in-time snapshot. - # Stamp the run so anything derived from it, an onboarding issue or a report, carries its own freshness signal and a reader can tell whether it still applies. run_utc = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") hub = subprocess.run( ["git", "rev-parse", "--short", "HEAD"], @@ -3556,7 +2338,6 @@ def main(argv=None): ) hub_sha = hub.stdout.strip() if hub.returncode == 0 else "unknown" - # --issue : emit only the convergence issue on stdout (title first line, then body) so it captures cleanly. if issue_mode: if len(repos) != 1: print("--issue takes exactly one cataloged repo", file=sys.stderr) @@ -3567,7 +2348,6 @@ def main(argv=None): except ShallowCloneError as e: print(e, file=sys.stderr) return 2 - # An unverifiable audit still produces an honest issue. except Exception as e: # noqa: BLE001 findings, audited_sha = [("ERROR", str(e))], "" title, body = render_issue( @@ -3596,9 +2376,7 @@ def main(argv=None): hard = 0 - # Fleet-wide, not per-repo, so it runs once and only on a full sweep: a name-filtered or - # --issue run is scoped to specific repos already in the registry and has nothing to gain - # from re-listing every repo the owner has. + # Fleet-wide, so it runs once and only on a full sweep. if not wanted: try: membership = membership_findings(spec) diff --git a/spec/audit_markdown.py b/spec/audit_markdown.py new file mode 100644 index 000000000..789ae4e5c --- /dev/null +++ b/spec/audit_markdown.py @@ -0,0 +1,654 @@ +"""The audit's Markdown reading and its README checks, read with markdown-it-py.""" + +import itertools +import json +import re +import urllib.error +import urllib.parse +import urllib.request + +import markdown_it +import validate +from carried_text import extract_section, normalize, strip_sections + + +def docker_hub_description(slug): + """The Docker Hub short description for a repo, or None if the image is absent (HTTP 404). + + A transient failure raises, so the caller reports it as unverified rather than passing. + """ + owner, repo = slug.split("/", 1) + url = f"https://hub.docker.com/v2/repositories/{owner.lower()}/{repo.lower()}/" + try: + with urllib.request.urlopen(url, timeout=5) as r: + return json.loads(r.read().decode("utf-8")).get("description") + except urllib.error.HTTPError as e: + if e.code == 404: + return None + raise + + +def undeclared_h2_headings(text, declared): + """Level-two headings in `text` that `declared` does not name, folded and sorted.""" + h2s = {h.strip().lower() for level, h, _, _ in _atx_headings(text) if level == 2} + return sorted(h2s - {d.strip().lower() for d in declared}) + + +# CommonMark plus tables, with escapes kept as their own tokens so a line can be rebuilt from its source. +_MD = markdown_it.MarkdownIt("commonmark").enable("table").disable("text_join") + + +def _atx_headings(text): + """`(level, raw text, first line, last line)` for each ATX heading, the form the region reader splits on.""" + tokens = _MD.parse(text) + return [ + (int(t.tag[1]), tokens[i + 1].content, t.map[0], t.map[1]) + for i, t in enumerate(tokens) + if t.type == "heading_open" and t.markup.startswith("#") and t.map + ] + + +def references(text): + """The reference definitions markdown-it reads from a document, keyed the way it resolves a link.""" + env = {} + _MD.parse(text, env) + return env.get("references", {}) + + +def heading_texts(markdown): + """Lowercased heading texts in a Markdown document, for case-insensitive section-presence matching.""" + return {h.strip().lower() for _, h, _, _ in _atx_headings(markdown)} + + +_HTML_COMMENT = re.compile(r"", re.DOTALL) + + +def strip_md_links(text, references=None): + """`text` with each Markdown link reduced to its label, everything else kept as written.""" + out = [] + for tok in _MD.parseInline(text, {"references": dict(references or {})})[0].children or []: + if tok.type in ("text", "html_inline"): + out.append(tok.content) + elif tok.type == "code_inline": + out.append(tok.markup + tok.content + tok.markup) + elif tok.type == "softbreak": + out.append("\n") + elif tok.type not in ("link_open", "link_close"): + out.append(tok.markup) + return "".join(out) + + +def title_and_intro(text): + """The H1 title text and the intro region between it and the first H2, HTML comments stripped.""" + lines = _HTML_COMMENT.sub("", normalize(text)).split("\n") + heads = _atx_headings("\n".join(lines)) + first = next((h for h in heads if h[0] == 1), None) + if first is None: + return None, "" + end = next((h[2] for h in heads if h[0] == 2 and h[2] > first[2]), len(lines)) + return first[1].strip(), "\n".join(ln.rstrip() for ln in lines[first[3] : end]).strip("\n") + + +def tagline(intro): + """The first line of a README/HISTORY intro region, the one canonical short description.""" + return intro.split("\n")[0] + + +# Text scans, since a parsed link keeps its target but not the reference name and position these checks read. +_MD_IMAGE_REF = re.compile(r"!\[[^\]]*\]\[([^\]]+)\]") + +_MD_IMAGE_INLINE = re.compile(r"!\[[^\]]*\]\((\S+?)\)") + +_LINK_DEF = re.compile(r"^\[([^\]]+)\]:\s*(\S+)", re.MULTILINE) + +# A URI scheme needs more than one character so a `C:` drive letter is not read as one. +_URI_SCHEME = re.compile(r"[a-z][a-z0-9+.\-]+:", re.IGNORECASE) + + +def unfenced_text(text): + """`text` normalized with every fenced code block's lines removed, since markup in a sample is shown rather than used.""" + lines = normalize(text).split("\n") + fenced = { + i for t in _MD.parse("\n".join(lines)) if t.type == "fence" and t.map for i in range(*t.map) + } + return "\n".join(ln for i, ln in enumerate(lines) if i not in fenced) + + +def readme_region(text, heading): + """extract_section against a comment-stripped, unfenced copy of the document, or None where the heading is absent.""" + return extract_section(_HTML_COMMENT.sub("", unfenced_text(text)), heading) + + +def shield_endpoints(region, defs): + """The image URLs a Markdown region renders, from its `![alt][ref]` and `![alt](url)` uses.""" + region = region or "" + urls = [defs[m.group(1)] for m in _MD_IMAGE_REF.finditer(region) if m.group(1) in defs] + return urls + [m.group(1) for m in _MD_IMAGE_INLINE.finditer(region)] + + +def shield_matches(url, shield): + """True where a rendered URL is the shield the model describes, by endpoint plus its query discriminators.""" + return ( + shield["match"] in url + and all(q in url for q in shield.get("requireQuery", [])) + and (shield.get("forbidQuery") is None or shield["forbidQuery"] not in url) + ) + + +def addressed_region(text, address): + """The region a shield's `in` address names, as `Heading` or `Heading > Sub-heading`, or None if absent.""" + parts = [p.strip() for p in address.split(">")] + body = readme_region(text, parts[0]) + if body is None or len(parts) == 1: + return body + lines = body.split("\n") + h3s = [h for h in _atx_headings(body) if h[0] == 3] + for i, (_, name, _, end) in enumerate(h3s): + if name.strip().lower() == parts[1].lower(): + stop = h3s[i + 1][2] if i + 1 < len(h3s) else len(lines) + return "\n".join(lines[end:stop]) + return None + + +def ordered_headings(markdown, level): + """Heading texts at `level` in document order, HTML comments stripped, fenced blocks skipped.""" + return [ + _HTML_COMMENT.sub("", h).strip() for lv, h, _, _ in _atx_headings(markdown) if lv == level + ] + + +def readme_section_findings(text, model, sel, public): + """README section presence and order against spec/readme-sections.json.""" + findings = [] + by_name = {s["name"].lower(): s for s in model["sections"]} + retired = {r.lower(): s for s in model["sections"] for r in s.get("retiredNames", [])} + heads = ordered_headings(text, 2) + lower = [h.lower() for h in heads] + present = set(lower) + + for h, hl in zip(heads, lower): + if hl in retired: + findings.append( + ( + "LETTER", + f"readme: section '{h}' uses a retired name - rename it to '{retired[hl]['name']}', which has no accepted aliases (spec/readme-structure.md)", + ) + ) + + for s in model["sections"]: + req = s["required"] + if req == "optional" or (req == "public" and not public): + continue + if {t.lower() for t in s.get("notApplicableTo", [])} & {t.lower() for t in sel}: + continue + if s["name"].lower() in present or any( + r.lower() in present for r in s.get("retiredNames", []) + ): + continue + why = " (a public repo; optional while private)" if req == "public" else "" + findings.append( + ( + "LETTER", + f"readme: no `## {s['name']}` section{why} - it is a required section (spec/readme-structure.md)", + ) + ) + + seq = [ + (by_name.get(hl) or retired.get(hl), h) + for h, hl in zip(heads, lower) + if hl in by_name or hl in retired + ] + for (prev, prev_h), (cur, cur_h) in itertools.pairwise(seq): + if cur["ordinal"] < prev["ordinal"]: + findings.append( + ( + "LETTER", + f"readme: section '{cur_h}' follows '{prev_h}' but is ordered before it - the declared sections keep their relative order (spec/readme-structure.md)", + ) + ) + + last = next((s for s in model["sections"] if s.get("last")), None) + if last and last["name"].lower() in present and lower[-1] != last["name"].lower(): + findings.append( + ( + "LETTER", + f"readme: `## {last['name']}` is not the last section, '{heads[-1]}' follows it - it closes the file, immediately before the link definitions (spec/readme-structure.md)", + ) + ) + + for s in model["sections"]: + if s["name"].lower() not in present or not s.get("subsections"): + continue + have = {h.lower() for h in ordered_headings(readme_region(text, s["name"]) or "", 3)} + for sub in s["subsections"]: + if sub.lower() not in have: + findings.append( + ( + "LETTER", + f"readme: `## {s['name']}` carries no `### {sub}` sub-section (spec/readme-structure.md)", + ) + ) + return findings + + +def distribution_prefixes(model, slug): + """The URL prefixes that make a link this repo's own, from the model's single declaration of ownership.""" + owner = slug.split("/")[0] + return tuple( + p.replace("{slug}", slug).replace("{owner}", owner).lower() + for p in model.get("distribution", {}).get("urlPrefixes", []) + ) + + +def link_kind(url, slug, is_rendered=False, prefixes=()): + """Which linkGroups kind a reference definition points at, for the repo at `slug`.""" + if url.startswith("#"): + return "anchor" + if not url.startswith(("http://", "https://")) and not _URI_SCHEME.match(url): + return "local" + if prefixes and url.lower().startswith(tuple(prefixes)): + return "distribution" + return "shield" if is_rendered else "external" + + +def canonical_link_entry(url, model, slug): + """The canonicalLinks entry `url` is a destination for, or None where the model fixes no name for it.""" + base = f"https://github.com/{slug}" + bare = url.rstrip("/") + for c in model.get("canonicalLinks", []): + if "match" in c: + if re.search(c["match"], url): + return c + continue + if "repoPath" not in c: + # Loud rather than skipped, since skipping would quietly stop enforcing this destination's name. + raise KeyError( + f"spec/readme-sections.json: canonicalLinks entry '{c.get('name')}' carries neither repoPath nor match" + ) + want = (base + c["repoPath"]).rstrip("/") + if ( + bare.lower().startswith(want.lower()) + if c.get("prefix") + else bare.lower() == want.lower() + ): + return c + return None + + +def canonical_name_findings(defs, model, slug, rendered=(), prefixes=()): + """Reference names against canonicalLinks, resolved over the whole definition set rather than one at a time.""" + findings, hits = [], {} + for ref, url in defs: + if link_kind(url, slug, ref in rendered, prefixes) != "distribution": + continue + c = canonical_link_entry(url, model, slug) + if c: + hits.setdefault(c["name"], (c, []))[1].append((ref, url)) + for name, (c, group) in hits.items(): + if c.get("perTarget") and len(group) > 1: + for ref, url in group: + if not (ref.endswith(f"-{name}") and len(ref) > len(name) + 1): + findings.append( + ( + "LETTER", + f"readme: the reference `[{ref}]` points at {url} - this repo publishes {len(group)} of these, so each is named `-{name}` (spec/readme-structure.md)", + ) + ) + else: + for ref, url in group: + if ref != name: + findings.append( + ( + "LETTER", + f"readme: the reference `[{ref}]` points at {url} and is named `{name}` in every repo - a shared destination carries a shared name (spec/readme-structure.md)", + ) + ) + return findings + + +def readme_link_findings(text, model, slug): + """Reference-definition naming and grouping, per the linkGroups, linkNaming and canonicalLinks model.""" + findings = [] + suffixes = {n["kind"]: n["suffix"] for n in model["linkNaming"]} + prefixes = distribution_prefixes(model, slug) + groups = model["linkGroups"] + declared = [g["name"] for g in groups] + holds = {g["name"].lower(): g["holds"] for g in groups} + + unfenced = unfenced_text(text) + seen_headers, current, in_group = [], None, {} + for ln in unfenced.split("\n"): + s = ln.strip() + h = re.fullmatch(r"", s) + if h and "omit from toc" not in h.group(1): + if not any( + h.group(1).lower().startswith(d.lower()) for d in model.get("directiveComments", []) + ): + current = h.group(1) + continue + d = _LINK_DEF.match(s) + if d: + if current is not None and current not in in_group: + seen_headers.append(current) + in_group.setdefault(current, []).append((d.group(1), d.group(2))) + + all_defs = [p for v in in_group.values() for p in v] + # A reference the document renders as an image is a shield whatever host serves it, so resolve those first. + by_ref = dict(all_defs) + rendered = {m.group(1) for m in _MD_IMAGE_REF.finditer(unfenced) if m.group(1) in by_ref} + described = {"shield": "a badge", "anchor": "an in-page anchor", "local": "a path in this repo"} + for ref, url in all_defs: + kind = link_kind(url, slug, ref in rendered, prefixes) + want = suffixes.get(kind, "") + got = "-shield" if ref.endswith("-shield") else "-link" if ref.endswith("-link") else "" + if got != want: + shown = f"`{want}`" if want else "a bare name with no suffix" + findings.append( + ( + "LETTER", + f"readme: the reference `[{ref}]` points at {described.get(kind, 'a URI')} and should end in {shown} (spec/readme-structure.md)", + ) + ) + findings += canonical_name_findings(all_defs, model, slug, rendered, prefixes) + + unknown = [h for h in seen_headers if h not in declared] + if unknown: + findings.append( + ( + "DRIFT", + f"readme: link-group header(s) outside the declared set: {', '.join(sorted(set(unknown)))} - the groups are {', '.join(declared)} (spec/readme-structure.md)", + ) + ) + ordered = [h for h in seen_headers if h in declared] + if ordered != sorted(ordered, key=declared.index): + findings.append( + ( + "DRIFT", + f"readme: the link groups run {', '.join(ordered)} rather than the declared order (spec/readme-structure.md)", + ) + ) + if in_group.get(None): + if not seen_headers: + findings.append( + ( + "DRIFT", + f"readme: the {len(in_group[None])} reference definitions carry no group headers - they are grouped under {', '.join(declared)} (spec/readme-structure.md)", + ) + ) + else: + findings.append( + ( + "DRIFT", + f"readme: {len(in_group[None])} reference definition(s) sit above the first group header (spec/readme-structure.md)", + ) + ) + for header, defs in in_group.items(): + if header is None or not defs: + continue + names = [r for r, _ in defs] + if names != sorted(names): + findings.append( + ( + "DRIFT", + f"readme: the `{header}` group is not sorted by reference name (spec/readme-structure.md)", + ) + ) + want_kind = holds.get(header.lower()) + strays = ( + {link_kind(u, slug, r in rendered, prefixes) for r, u in defs} - {want_kind} + if want_kind + else set() + ) + if strays: + findings.append( + ( + "DRIFT", + f"readme: the `{header}` group holds {', '.join(sorted(strays))} reference(s) where it holds {want_kind} (spec/readme-structure.md)", + ) + ) + return findings + + +# A tool row is read as text for its reference name, with the outer pipes optional as GitHub's Markdown makes them. +_TOOL_ROW = re.compile(r"^\|?\s*\[([^\]]+)\]\[([^\]]+)\]\s*\|\s*([^|]*?)\s*(?:\||$)") + +_TOOL_BULLET = re.compile(r"^[-*]\s*\[([^\]]+)\]\[([^\]]+)\]\s*(.*)$") + + +def third_party_tool_findings(text, catalog): + """A repo's 3rd Party Tools entries against the shared catalog in spec/third-party-tools.json.""" + body = readme_region(text, "3rd Party Tools") + if body is None: + return [] # the absent section is already one finding from readme_section_findings + declared = {t["name"].lower(): t for t in catalog["tools"]} + # Read over the unfenced text, as readme_link_findings does: a sample footnote is not a definition. + defs = {m.group(1): m.group(2) for m in _LINK_DEF.finditer(unfenced_text(text))} + findings = [] + listed = [] + in_head = False + for tok in _MD.parse(body): + if tok.type in ("thead_open", "thead_close"): + in_head = tok.type == "thead_open" + elif in_head and tok.type == "inline" and tok.content.strip().lower() == "license": + findings.append( + ( + "LETTER", + "readme: 3rd Party Tools carries a License column - a license belongs to the dependency and is authoritative at its source, so restating it adds a maintenance obligation and no information (spec/readme-structure.md)", + ) + ) + for ln in body.split("\n"): + m = _TOOL_ROW.match(ln.strip()) or _TOOL_BULLET.match(ln.strip()) + if not m: + continue + name, ref, desc = m.group(1), m.group(2), m.group(3).strip(" -:") + listed.append(name) + want = declared.get(name.lower()) + if want is None: + continue + url = defs.get(ref) + if url is not None and url.rstrip("/") != want["link"].rstrip("/"): + findings.append( + ( + "LETTER", + f"readme: 3rd Party Tools links {name} as {url} where the fleet links it as {want['link']} - a shared tool carries one link (spec/third-party-tools.json)", + ) + ) + if desc != want["description"]: + shown = f"'{desc}'" if desc else "no description" + findings.append( + ( + "LETTER", + f"readme: 3rd Party Tools describes {name} as {shown} where the fleet describes it as '{want['description']}' - a shared tool carries one description (spec/third-party-tools.json)", + ) + ) + # Ordering covers every tool listed, not only the cataloged ones, since a reader scans the whole list. + if listed != sorted(listed, key=str.lower): + first = next( + f"{a} before {b}" for a, b in itertools.pairwise(listed) if a.lower() > b.lower() + ) + findings.append( + ( + "LETTER", + f"readme: 3rd Party Tools is not alphabetized ({first}) - the list is scanned rather than read (spec/readme-structure.md)", + ) + ) + return findings + + +def readme_shield_findings(text, model, entry): + """Shield presence, by the additive classes in spec/readme-sections.json.""" + findings = [] + unfenced = unfenced_text(text) + defs = {m.group(1): m.group(2) for m in _LINK_DEF.finditer(unfenced)} + # A retired badge service is scanned across the whole document rather than per section, since a dead badge is wrong wherever it sits. + rendered_urls = set(shield_endpoints(unfenced, defs)) + used_refs = {m.group(1) for m in _MD_IMAGE_REF.finditer(unfenced)} + for dep in model.get("deprecatedShields", []): + defined_urls = set() + for ref, url in sorted(defs.items()): + if dep["match"] in url: + defined_urls.add(url) + if ref in used_refs: + verb = f"renders {dep['label']}" + elif url in rendered_urls: + verb = f"defines {dep['label']} and it is rendered elsewhere" + else: + verb = f"defines {dep['label']} and nothing renders it" + findings.append( + ( + "LETTER", + f"readme: `[{ref}]` {verb}, which is retired - {dep['reason']} (spec/readme-structure.md)", + ) + ) + for url in sorted({u for u in rendered_urls if dep["match"] in u} - defined_urls): + findings.append( + ( + "LETTER", + f"readme: an inline image renders {dep['label']}, which is retired - {dep['reason']} (spec/readme-structure.md)", + ) + ) + targets = {(p.get("target") if isinstance(p, dict) else p) for p in entry.get("publish", [])} + secrets = set(entry.get("requiredSecrets", [])) + want = [] + for cls in model["shieldClasses"]: + t = cls["trigger"] + if ( + t["kind"] == "always" + or (t["kind"] == "publish" and t.get("target") in targets) + or (t["kind"] == "secret" and t.get("name") in secrets) + ): + want += cls["shields"] + for sh in want: + region = addressed_region(text, sh["in"]) + if region is None: + continue + if not any(shield_matches(u, sh) for u in shield_endpoints(region, defs)): + queries = "".join(f" with `{q}`" for q in sh.get("requireQuery", [])) + if sh.get("forbidQuery") is not None: + queries += f" without `{sh['forbidQuery']}`" + findings.append( + ( + "LETTER", + f"readme: `{sh['in']}` carries no {sh['label']} shield ({sh['match']}{queries}), which this repo's deliverables require (spec/readme-structure.md)", + ) + ) + if not sh.get("exclusive"): + continue + # Rendered elsewhere, which for the license shield is the whole point: it belongs at the bottom and nowhere else. + outside = strip_sections( + _HTML_COMMENT.sub("", unfenced_text(text)), [sh["in"].split(">")[0].strip()] + ) + if any(shield_matches(u, sh) for u in shield_endpoints(outside, defs)): + findings.append( + ( + "LETTER", + f"readme: the {sh['label']} shield is rendered outside `{sh['in']}` - it belongs there and nowhere else (spec/readme-structure.md)", + ) + ) + return findings + + +def description_findings(doc_texts, entry, live, slug): + """The README title/tagline, the GitHub About description, and the Docker Hub short description, checked as one mirror set.""" + findings = [] + declared = None + if "description" in entry: + # Delegates to validate.py's own contract rather than a second copy of it. + shape_errors = validate.description_errors(slug, entry["description"]) + if shape_errors: + findings += [ + ( + "DEFECT", + f"{msg}. Treated as undeclared here, since it fails spec/validate.py's contract.", + ) + for msg in shape_errors + ] + else: + declared = entry["description"] + readme_want = None + if "README.md" in doc_texts: + title, intro = title_and_intro(doc_texts["README.md"]) + intro_line = tagline(intro) + refs = references(doc_texts["README.md"]) + # The H1 is the repository name, and a hyphenated name may render its hyphens as spaces. + repo_name = live.get("name") or slug.split("/")[-1] + if not title: + findings.append( + ( + "LETTER", + "readme: no `# ` H1 title - the README opens with `# ` then the tagline (spec/readme-structure.md)", + ) + ) + elif title.replace("-", " ") != repo_name.replace("-", " "): + findings.append( + ( + "LETTER", + f"readme: the H1 title '{title}' is not the repo name '{repo_name}' (a hyphenated name may render its hyphens as spaces) - the H1 is the repository name (spec/readme-structure.md)", + ) + ) + if not intro_line: + findings.append( + ( + "LETTER", + "readme: no tagline after the H1 - the README opens with the title then a one-line description, which doubles as the About description (spec/readme-structure.md)", + ) + ) + else: + if strip_md_links(intro_line, refs) != intro_line: + findings.append( + ( + "LETTER", + "readme: the tagline carries Markdown links - keep it link-free plain text, it doubles as the repo About description (spec/readme-structure.md)", + ) + ) + readme_want = strip_md_links(intro_line, refs).strip() + if len(readme_want) > 100: + findings.append( + ( + "LETTER", + f"readme: the tagline is {len(readme_want)} characters, over the 100-char limit (Docker Hub's short-description cap, the tightest surface it feeds) - tighten it to one short sentence (spec/readme-structure.md)", + ) + ) + # The declared field wins once a repo has one, and every mirror is measured against it. + want = declared or readme_want + if want is None: + return findings + source = "registry/repos.json" if declared else "the README" + if declared and readme_want is not None and readme_want != declared: + findings.append( + ( + "LETTER", + f"readme: the tagline ('{readme_want}') does not match the declared description ('{declared}') - registry/repos.json's description is canonical once declared, so the README follows it (GOVERNANCE.md Repository Details)", + ) + ) + desc = (live.get("description") or "").strip() + if desc != want: + findings.append( + ( + "LETTER", + f"description: the About description does not match the {'declared description' if declared else 'README tagline'} (description '{desc}' vs '{want}') - set it from {source}, or sharpen {source} first if the description carries real detail (GOVERNANCE.md Repository Details)", + ) + ) + # Docker Hub short description mirrors the same canonical value, for a repo that publishes a docker image. + if any( + (pt.get("target") if isinstance(pt, dict) else pt) == "docker" + for pt in entry.get("publish", []) + ): + try: + dh = docker_hub_description(slug) + except Exception as e: # noqa: BLE001 + dh = None + findings.append( + ( + "DRIFT", + f"description: could not read the Docker Hub short description to verify it mirrors {source} ({e}) - verify by hand", + ) + ) + if dh is not None and dh.strip() != want: + findings.append( + ( + "LETTER", + f"description: the Docker Hub short description ('{dh.strip()}') does not match '{want}' - set it from {source} (spec/readme-structure.md)", + ) + ) + return findings diff --git a/tests/test_audit_helpers.py b/tests/test_audit_helpers.py index 488ba6303..eb65d4ab0 100755 --- a/tests/test_audit_helpers.py +++ b/tests/test_audit_helpers.py @@ -20,6 +20,7 @@ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent / "spec")) import audit +import audit_markdown from git_env import git_location_cleared _LOCATION = contextlib.ExitStack() @@ -1133,7 +1134,7 @@ def test_unfenced_text(self) -> None: ] for label, doc, want in cases: with self.subTest(label): - self.assertEqual(audit.unfenced_text(doc), want) + self.assertEqual(audit_markdown.unfenced_text(doc), want) def test_undeclared_h2_headings(self) -> None: cases = [ @@ -1192,7 +1193,7 @@ def test_undeclared_h2_headings(self) -> None: ] for label, doc, declared, want in cases: with self.subTest(label): - self.assertEqual(audit.undeclared_h2_headings(doc, declared), want) + self.assertEqual(audit_markdown.undeclared_h2_headings(doc, declared), want) class ScanCase(unittest.TestCase): @@ -1339,9 +1340,15 @@ def test_title_and_intro_mirror(self) -> None: r_two = "# W\n\nLine one.\n\nLine two.\n\n## Build\n" h_joined = "# W\n\nLine one.\nLine two.\n\n## Release History\n" msg = "history: README/HISTORY title+intro mirror detection" - self.assertEqual(audit.title_and_intro(r_md), audit.title_and_intro(h_md), msg) - self.assertNotEqual(audit.title_and_intro(r_md), audit.title_and_intro(h_bad), msg) - self.assertNotEqual(audit.title_and_intro(r_two), audit.title_and_intro(h_joined), msg) + self.assertEqual( + audit_markdown.title_and_intro(r_md), audit_markdown.title_and_intro(h_md), msg + ) + self.assertNotEqual( + audit_markdown.title_and_intro(r_md), audit_markdown.title_and_intro(h_bad), msg + ) + self.assertNotEqual( + audit_markdown.title_and_intro(r_two), audit_markdown.title_and_intro(h_joined), msg + ) def test_strip_md_links(self) -> None: linked = "Utility to clean [media](https://x.example/Foo_(bar)) per the [spec][spec-ref]." @@ -1349,22 +1356,28 @@ def test_strip_md_links(self) -> None: inner_link = "See [[docs](url)] for details." escaped_bracket = r"See [API \[docs](url) for details." msg = "description: strip_md_links behavior" - refs = audit.references("x\n\n[spec-ref]: https://example.test\n") + refs = audit_markdown.references("x\n\n[spec-ref]: https://example.test\n") self.assertEqual( - audit.strip_md_links(linked, refs), "Utility to clean media per the spec.", msg + audit_markdown.strip_md_links(linked, refs), "Utility to clean media per the spec.", msg ) self.assertEqual( - audit.strip_md_links(linked), "Utility to clean media per the [spec][spec-ref].", msg + audit_markdown.strip_md_links(linked), + "Utility to clean media per the [spec][spec-ref].", + msg, + ) + self.assertEqual( + audit_markdown.strip_md_links("Plain intro line."), "Plain intro line.", msg + ) + self.assertEqual(audit_markdown.strip_md_links(nested), "See API [docs] for details.", msg) + self.assertEqual(audit_markdown.strip_md_links(inner_link), "See [docs] for details.", msg) + self.assertEqual( + audit_markdown.strip_md_links(escaped_bracket), r"See API \[docs for details.", msg ) - self.assertEqual(audit.strip_md_links("Plain intro line."), "Plain intro line.", msg) - self.assertEqual(audit.strip_md_links(nested), "See API [docs] for details.", msg) - self.assertEqual(audit.strip_md_links(inner_link), "See [docs] for details.", msg) - self.assertEqual(audit.strip_md_links(escaped_bracket), r"See API \[docs for details.", msg) def test_strip_md_links_stays_linear(self) -> None: pathological = "[" * 20000 start = time.monotonic() - audit.strip_md_links(pathological) + audit_markdown.strip_md_links(pathological) elapsed = time.monotonic() - start self.assertLessEqual( elapsed, @@ -2497,7 +2510,7 @@ def setUp(self) -> None: def test_section_cases(self) -> None: for label, text, sel_types, public, wantn in README_CASES: with self.subTest(label): - got = audit.readme_section_findings(text, self.rm, sel_types, public) + got = audit_markdown.readme_section_findings(text, self.rm, sel_types, public) self.assertEqual(len(got), wantn, f"readme sections: {label} -> {got}") def test_shield_cases(self) -> None: @@ -2505,7 +2518,7 @@ def test_shield_cases(self) -> None: label, text, ent, wantn = case[:4] want_text = case[4] if len(case) > 4 else None with self.subTest(label): - got = audit.readme_shield_findings(text, self.rm, ent) + got = audit_markdown.readme_shield_findings(text, self.rm, ent) good = len(got) == wantn and ( want_text is None or any(want_text in t for _, t in got) ) @@ -2514,7 +2527,7 @@ def test_shield_cases(self) -> None: def test_link_cases(self) -> None: for label, text, want_letter, want_drift in LINK_CASES: with self.subTest(label): - got = audit.readme_link_findings(text, self.rm, "o/r") + got = audit_markdown.readme_link_findings(text, self.rm, "o/r") gl = sum(1 for k, _ in got if k == "LETTER") gd = sum(1 for k, _ in got if k == "DRIFT") self.assertEqual( @@ -2525,7 +2538,7 @@ def test_tool_cases(self) -> None: cat = audit.load("spec/third-party-tools.json") for label, text, wantn in TOOL_CASES: with self.subTest(label): - got = audit.third_party_tool_findings(text, cat) + got = audit_markdown.third_party_tool_findings(text, cat) self.assertEqual(len(got), wantn, f"3rd party tools: {label} -> {got}") def test_canonical_link_entry_with_neither_repopath_nor_match_raises(self) -> None: @@ -2536,7 +2549,7 @@ def test_canonical_link_entry_with_neither_repopath_nor_match_raises(self) -> No KeyError, msg="canonicalLinks entry with neither repoPath nor match: no error raised", ) as ctx: - audit.canonical_link_entry("https://example.test/", broken, "o/r") + audit_markdown.canonical_link_entry("https://example.test/", broken, "o/r") message = str(ctx.exception) self.assertTrue( "readme-sections.json" in message and "orphan-link" in message, @@ -2544,11 +2557,11 @@ def test_canonical_link_entry_with_neither_repopath_nor_match_raises(self) -> No ) def test_tagline_is_the_first_line_of_the_intro_region(self) -> None: - two_para = audit.title_and_intro( + two_para = audit_markdown.title_and_intro( "# X\n\nThe tagline.\n\nA clarifying paragraph.\n\n## Next\n" )[1] - self.assertEqual(audit.tagline(two_para), "The tagline.") - self.assertNotIn("\n", audit.tagline(two_para)) + self.assertEqual(audit_markdown.tagline(two_para), "The tagline.") + self.assertNotIn("\n", audit_markdown.tagline(two_para)) class DescriptionCase(unittest.TestCase): @@ -2557,11 +2570,13 @@ class DescriptionCase(unittest.TestCase): def test_description_cases(self) -> None: for label, doc_texts_fx, entry_fx, live_fx, wantn in DESC_CASES: with self.subTest(label): - got = audit.description_findings(doc_texts_fx, entry_fx, live_fx, "owner/Fixture") + got = audit_markdown.description_findings( + doc_texts_fx, entry_fx, live_fx, "owner/Fixture" + ) self.assertEqual(len(got), wantn, f"description: {label} -> {got}") def test_null_declared_field_is_a_defect_via_validates_contract(self) -> None: - null_declared = audit.description_findings( + null_declared = audit_markdown.description_findings( DESC_README, {"description": None}, {"description": "A short tagline."}, @@ -2576,7 +2591,7 @@ def test_null_declared_field_is_a_defect_via_validates_contract(self) -> None: ) def test_declared_field_defect_names_the_repo(self) -> None: - null_declared = audit.description_findings( + null_declared = audit_markdown.description_findings( DESC_README, {"description": None}, {"description": "A short tagline."}, @@ -2588,7 +2603,7 @@ def test_declared_field_defect_names_the_repo(self) -> None: ) def test_declared_field_names_the_registry_as_the_source(self) -> None: - declared_mismatch = audit.description_findings( + declared_mismatch = audit_markdown.description_findings( DESC_README, {"description": "The declared description."}, {"description": "Something else."}, @@ -2604,9 +2619,9 @@ def test_declared_field_names_the_registry_as_the_source(self) -> None: def test_stale_docker_hub_short_description_is_reported(self) -> None: with mock.patch.object( - audit, "docker_hub_description", lambda slug: "A stale Docker Hub blurb." + audit_markdown, "docker_hub_description", lambda slug: "A stale Docker Hub blurb." ): - docker_mismatch = audit.description_findings( + docker_mismatch = audit_markdown.description_findings( DESC_README, {"description": "A short tagline.", "publish": [{"target": "docker"}]}, {"description": "A short tagline."}, diff --git a/tests/test_audit_size.py b/tests/test_audit_size.py new file mode 100755 index 000000000..c8a7cf03b --- /dev/null +++ b/tests/test_audit_size.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +"""Hold the audit engine and its Markdown module under their size ceilings.""" + +from __future__ import annotations + +import unittest +from pathlib import Path + +SPEC = Path(__file__).resolve().parents[1] / "spec" +CEILINGS = {"audit.py": 3000, "audit_markdown.py": 1000} + + +class AuditSizeCase(unittest.TestCase): + """Tests live in tests/ and parsing is a library's, so neither file has a reason to grow back.""" + + def test_each_file_stays_under_its_ceiling(self) -> None: + for name, ceiling in CEILINGS.items(): + with self.subTest(name=name): + lines = len((SPEC / name).read_text(encoding="utf-8").splitlines()) + self.assertLessEqual(lines, ceiling, f"{name} has {lines} lines") + + +if __name__ == "__main__": + unittest.main() From 425233bab10596dfb45ecb46cd87c1e8e21b1fd7 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 23 Oct 2026 15:05:58 -0700 Subject: [PATCH 05/20] Run the Audit Helper Suite Under Coverage in the Validate Action The validate action's coverage step runs tests/test_audit_helpers.py in place of the audit's --selftest, with tests/requirements.txt installed, since uvx alone reads no script's inline metadata. workflow_reuse.py's self-test gets the same libraries, and OPERATIONS.md's local commands match. --- .github/actions/validate/action.yml | 8 ++++---- OPERATIONS.md | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/actions/validate/action.yml b/.github/actions/validate/action.yml index 64e605422..06941538d 100644 --- a/.github/actions/validate/action.yml +++ b/.github/actions/validate/action.yml @@ -23,14 +23,14 @@ runs: python-version: "3.13" # The unittest suite under tests/ is the validator's Python leg, since the hub declares '.' in python-directories, so it runs there under coverage and uploads to Codecov like any adopter's. - # What stays here is each self-test that is not a unittest suite, and the agent-safety kit carries its own tests because it ships as a unit. - # Every run after the first appends, so one report covers these self-tests, informational with no threshold adopted. + # What stays here is the audit's helper suite and each self-test that is not a unittest suite, and the agent-safety kit carries its own tests because it ships as a unit. + # The audit's libraries come from tests/requirements.txt, since uvx alone reads no script's inline metadata. - name: Run script self-tests step shell: bash run: | set -Eeuo pipefail - uvx coverage@latest run --source=scripts,spec,host-setup spec/audit.py --selftest - uvx coverage@latest run --source=scripts,spec,host-setup --append spec/workflow_reuse.py --selftest + uvx --with-requirements tests/requirements.txt coverage@latest run --source=scripts,spec,host-setup -m unittest discover -s tests -p test_audit_helpers.py + uvx --with-requirements tests/requirements.txt coverage@latest run --source=scripts,spec,host-setup --append spec/workflow_reuse.py --selftest uvx coverage@latest run --source=scripts,spec,host-setup --append host-setup/agent-safety/claude/gh-write-guard.py --selftest uvx coverage@latest run --source=scripts,spec,host-setup --append host-setup/agent-safety/claude/test_install.py uvx coverage@latest report diff --git a/OPERATIONS.md b/OPERATIONS.md index 1e8b90187..b35d9e08d 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -22,8 +22,8 @@ uvx ruff@latest check . uvx ruff@latest format --check . uvx mypy@latest uvx --with-requirements tests/requirements.txt coverage@latest run --source=scripts,spec,host-setup -m unittest discover -s tests -uvx coverage@latest run --source=scripts,spec,host-setup --append spec/audit.py --selftest -uvx coverage@latest run --source=scripts,spec,host-setup --append spec/workflow_reuse.py --selftest +uvx --with-requirements tests/requirements.txt coverage@latest run --source=scripts,spec,host-setup --append spec/audit.py --selftest +uvx --with-requirements tests/requirements.txt coverage@latest run --source=scripts,spec,host-setup --append spec/workflow_reuse.py --selftest uvx coverage@latest run --source=scripts,spec,host-setup --append host-setup/agent-safety/claude/gh-write-guard.py --selftest uvx coverage@latest run --source=scripts,spec,host-setup --append host-setup/agent-safety/claude/test_install.py uvx coverage@latest report From 0113be7c8540fb95183518e2fb0e4ecd10ce70c9 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 23 Oct 2026 15:23:39 -0700 Subject: [PATCH 06/20] Fix Local Review Findings on the Audit Refactor A forbidden token anchored on a newline matches a job's first key again, since a job's token view opens with its key line. A driftNote naming a registered check says the audit evaluates it. spec/validate.py refuses a file check with no baseline check before it in its run. The invalid-byte probe's platform flag lives in the test module, and OPERATIONS.md names the suite CI runs of the audit. --- .github/actions/validate/action.yml | 3 ++- OPERATIONS.md | 4 ++-- scripts/carry.py | 6 +++--- spec/audit.py | 14 +++++++++----- spec/carried_text.py | 6 +++--- spec/validate.py | 5 +++++ tests/test_audit_helpers.py | 17 +++++++++++++++-- tests/test_spec_validate.py | 1 + 8 files changed, 40 insertions(+), 16 deletions(-) diff --git a/.github/actions/validate/action.yml b/.github/actions/validate/action.yml index 06941538d..55e0de961 100644 --- a/.github/actions/validate/action.yml +++ b/.github/actions/validate/action.yml @@ -23,7 +23,8 @@ runs: python-version: "3.13" # The unittest suite under tests/ is the validator's Python leg, since the hub declares '.' in python-directories, so it runs there under coverage and uploads to Codecov like any adopter's. - # What stays here is the audit's helper suite and each self-test that is not a unittest suite, and the agent-safety kit carries its own tests because it ships as a unit. + # What stays here is the audit's helper suite and each self-test that is not a unittest suite. + # The agent-safety kit carries its own tests because it ships as a unit. # The audit's libraries come from tests/requirements.txt, since uvx alone reads no script's inline metadata. - name: Run script self-tests step shell: bash diff --git a/OPERATIONS.md b/OPERATIONS.md index b35d9e08d..37a6e8c16 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -4,7 +4,7 @@ How this repository is run. It ships no application code, so its operations are ## Local Verification -What verifying a change here requires, including the part CI cannot perform. The gates below do run in CI, so a local run of them buys an earlier failure rather than a different one, with two exceptions stated where each belongs, the prose gate in the next section and `test_install.py` below the block. The two verifications CI never performs are the fleet audit and the repository-configuration check, because each reads live state in another repository over the API and a pull request runner is given neither the credentials nor a target to read. `python3 spec/audit.py --selftest` is what CI runs of the audit, which exercises the checker against its fixtures and reads no repository at all. So a change to [spec/](./spec/), [registry/](./registry/), [repo-config/](./repo-config/) or [AUDIT.md](./AUDIT.md) is verified by running `uv run --no-project --script spec/audit.py` and `repo-config/configure.sh check` from this checkout before the pull request opens, each from the repository root and each taking the arguments the two runbooks below give it. A green pipeline says nothing about either, and taking it as coverage is the failure this section exists to name. +What verifying a change here requires, including the part CI cannot perform. The gates below do run in CI, so a local run of them buys an earlier failure rather than a different one, with two exceptions stated where each belongs, the prose gate in the next section and `test_install.py` below the block. The two verifications CI never performs are the fleet audit and the repository-configuration check, because each reads live state in another repository over the API and a pull request runner is given neither the credentials nor a target to read. `tests/test_audit_helpers.py` is what CI runs of the audit, which exercises the checker against its fixtures and reads no repository at all. So a change to [spec/](./spec/), [registry/](./registry/), [repo-config/](./repo-config/) or [AUDIT.md](./AUDIT.md) is verified by running `uv run --no-project --script spec/audit.py` and `repo-config/configure.sh check` from this checkout before the pull request opens, each from the repository root and each taking the arguments the two runbooks below give it. A green pipeline says nothing about either, and taking it as coverage is the failure this section exists to name. ### Run the gates locally, which is wider than CI @@ -22,7 +22,7 @@ uvx ruff@latest check . uvx ruff@latest format --check . uvx mypy@latest uvx --with-requirements tests/requirements.txt coverage@latest run --source=scripts,spec,host-setup -m unittest discover -s tests -uvx --with-requirements tests/requirements.txt coverage@latest run --source=scripts,spec,host-setup --append spec/audit.py --selftest +uvx --with-requirements tests/requirements.txt coverage@latest run --source=scripts,spec,host-setup --append -m unittest discover -s tests -p test_audit_helpers.py uvx --with-requirements tests/requirements.txt coverage@latest run --source=scripts,spec,host-setup --append spec/workflow_reuse.py --selftest uvx coverage@latest run --source=scripts,spec,host-setup --append host-setup/agent-safety/claude/gh-write-guard.py --selftest uvx coverage@latest run --source=scripts,spec,host-setup --append host-setup/agent-safety/claude/test_install.py diff --git a/scripts/carry.py b/scripts/carry.py index e5612487e..47856aa5d 100755 --- a/scripts/carry.py +++ b/scripts/carry.py @@ -393,9 +393,9 @@ def verify_target( def carried_text_module() -> Any: """`spec/carried_text.py`, imported on first use rather than at module import, then held. - It is a script rather than a package module and imports its own sibling by bare name, so the - spec directory has to be importable before it is. Lazily, so a tree carry never pays to load - the audit for a reader it does not use. Cached because `fence_step` reaches for it once per + It is a script-directory module rather than a package one, so the spec directory has to be + importable before it is. Lazily, so a tree carry never pays to load it for a reader it does + not use. Cached because `fence_step` reaches for it once per line: locating this repository's own `GOVERNANCE.md` sections costs 28.8 ms of `sys.path` scanning and `sys.modules` lookups uncached against 0.9 ms cached, over 7326 calls, and a re-vendor walks each file several times. diff --git a/spec/audit.py b/spec/audit.py index 759defc06..fad30b10d 100755 --- a/spec/audit.py +++ b/spec/audit.py @@ -698,6 +698,13 @@ def driftnote_findings(entry, spec, open_count): f"registry: driftNote names check '{cid}', which neither spec/checks.json nor spec/project-types.json defines - fix the id or drop the note: {quoted}", ) ) + elif any(c["id"] == cid for c in registered_checks()): + out.append( + ( + "DRIFT", + f"registry: driftNote names check '{cid}', which this audit evaluates (spec/checks.json) - delete the note once that check passes: {quoted}", + ) + ) elif owner and owner not in declared_types(entry): out.append( ( @@ -927,7 +934,7 @@ def check_interface(path, contract, text): # An absent job is already a requiredJobKeys finding, so its tokens are not reported again. for job, toks in contract.get("requireTokensInJob", {}).items(): if job in jobs: - view = code_view(jobs[job], 2) + view = f" {job}:\n{code_view(jobs[job], 2)}" for t in toks: if t not in view: findings.append( @@ -935,7 +942,7 @@ def check_interface(path, contract, text): ) for job, toks in contract.get("forbidTokensInJob", {}).items(): if job in jobs: - view = code_view(jobs[job], 2) + view = f" {job}:\n{code_view(jobs[job], 2)}" for t in toks: if t in view: findings.append( @@ -2163,9 +2170,6 @@ def check_registry_driftnotes(run): return driftnote_findings(run.entry, run.spec, len(run.findings)) -_IS_WINDOWS = sys.platform == "win32" - - def _selftest(): """Run the self-test with no variable naming a repository in the environment.""" with git_location_cleared(): diff --git a/spec/carried_text.py b/spec/carried_text.py index 9a537e4e1..d88d779cf 100644 --- a/spec/carried_text.py +++ b/spec/carried_text.py @@ -31,8 +31,8 @@ def normalize(text): def fence_step(line, marker, marker_len): """One line's effect on fence state `(marker, marker_len)`, as `(marker, marker_len, is_boundary)`. - CommonMark fences: at most 3 leading spaces, a closer of the opener's character at least as long - with nothing after it, and no backtick in a backtick fence's info string. + CommonMark fences: an opener has at most 3 leading spaces. A closer repeats the opener's character + at least as many times, with nothing after it. A backtick fence's info string holds no backtick. """ stripped = line.lstrip(" ") indent = len(line) - len(stripped) @@ -58,7 +58,7 @@ def _h2_text(line, marker, boundary): def extract_section(text, heading): """The normalized `## ` region, heading line included, up to the next H2, or None. - The heading is found by its folded text, and its exact bytes are part of the region, so a re-cased + The heading is found by its folded text. Its exact bytes are part of the region, so a re-cased heading is found and still reads as drift. """ want = heading.strip().lower() diff --git a/spec/validate.py b/spec/validate.py index 9a31de633..6604314cd 100755 --- a/spec/validate.py +++ b/spec/validate.py @@ -396,11 +396,16 @@ def check_registry_errors(registry, types, repos): return ["checks.json: 'checks' must be an array"] errors = [] ids = set() + read = False for i, check in enumerate(checks): where = f"checks.json: checks[{i}]" if not isinstance(check, dict): errors.append(f"{where} must be an object") continue + # A file check runs only on what a baseline check before it in the same run has read. + read = check.get("scope") == "baseline" or (read and check.get("scope") == "file") + if check.get("scope") == "file" and not read: + errors.append(f"{where} is a file check with no baseline check before it in its run") cid = check.get("id") if not isinstance(cid, str) or not CHECK_ID_RE.fullmatch(f"({cid})"): errors.append(f"{where} id {cid!r} is not a dotted lowercase check id") diff --git a/tests/test_audit_helpers.py b/tests/test_audit_helpers.py index eb65d4ab0..4129a477e 100755 --- a/tests/test_audit_helpers.py +++ b/tests/test_audit_helpers.py @@ -24,6 +24,7 @@ from git_env import git_location_cleared _LOCATION = contextlib.ExitStack() +IS_WINDOWS = sys.platform == "win32" def setUpModule() -> None: @@ -583,6 +584,18 @@ def test_finding_counts(self) -> None: self.assertEqual(len(audit.check_interface("wf", contract, text)), want) +class ForbiddenTokenCase(unittest.TestCase): + """A forbidden token anchored on a newline matches the job's first key too.""" + + def test_job_level_environment_as_the_first_key(self) -> None: + text = ( + "jobs:\n deploy:\n environment: production\n" + " uses: acme/hub/.github/workflows/deploy-site-task.yml@x\n" + ) + contract = {"forbidTokensInJob": {"deploy": ["\n environment:"]}} + self.assertEqual(len(audit.check_interface("wf", contract, text)), 1) + + class SplitJobsCase(unittest.TestCase): """Holds the split_jobs edge cases.""" @@ -917,7 +930,7 @@ def test_non_utf8_filename_round_trips(self) -> None: try: self.run_git(root, "add", "--", bad_name) except subprocess.CalledProcessError: - if not audit._IS_WINDOWS: + if not IS_WINDOWS: raise self.skipTest("hub_tracked: non-UTF-8 filename (git cannot add it)") self.run_git(root, "commit", "-q", "-m", "add invalid-utf8 name") @@ -958,7 +971,7 @@ def probe(self, run: Any, windows: bool) -> BaseException: case = HubTrackedCase("test_non_utf8_filename_round_trips") self.addCleanup(case.doCleanups) with ( - mock.patch.object(audit, "_IS_WINDOWS", windows), + mock.patch.object(sys.modules[__name__], "IS_WINDOWS", windows), mock.patch.object(subprocess, "run", run), ): try: diff --git a/tests/test_spec_validate.py b/tests/test_spec_validate.py index e173ea48f..279268885 100755 --- a/tests/test_spec_validate.py +++ b/tests/test_spec_validate.py @@ -66,6 +66,7 @@ def test_each_malformed_field_is_reported(self) -> None: "endsWithType", ), ("a check that is no object", ["a.b"], "must be an object"), + ("a file check with nothing read", [self.check(scope="file")], "no baseline check"), ] for label, checks, needle in cases: with self.subTest(label): From d92bb4e7e5edc73f46204291a2f3877bdcba5997 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 23 Oct 2026 15:50:42 -0700 Subject: [PATCH 07/20] Fix Second-Pass Review Findings on the Audit Refactor An unparseable workflow is one interface finding, and the concurrency check no longer adds advice for a syntax error. The ambient GIT_DIR test runs the helper suite, since the smoke self-test runs no git. AUDIT.md names spec/checks.json where the runner resolves a driftNote id, and the stale comments and docstrings the moved self-test left are updated. --- AUDIT.md | 4 ++-- spec/audit.py | 11 ++--------- spec/validate.py | 5 ++--- spec/workflow_reuse.py | 4 ++-- tests/test_audit_helpers.py | 6 ++++++ tests/test_git_env.py | 17 +++++++++++++---- tests/test_spec_validate.py | 3 +-- 7 files changed, 28 insertions(+), 22 deletions(-) diff --git a/AUDIT.md b/AUDIT.md index 636295ed0..eb1113980 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -69,7 +69,7 @@ Which carried files and sections a repo is expected to have is decided by its sc For each applicable type in [`spec/project-types.json`][project-types] and every cross-cutting dimension, evaluate each check at its stated verdict tier: -**Every check under a project type is judged by hand. The cross-cutting dimensions are only partly mechanized, and the line between the two halves is not where a reader assumes.** [`spec/audit.py`][audit-runner] evaluates **no** check belonging to a type in `spec/project-types.json`, and it reads that file for one purpose only, to resolve the id a registry `driftNote` names (section 8) against the catalog and against the repo's declared types. Resolving an id is not running the check it names. What the runner does mechanize is the deterministic subset in section 0, and that subset lands on several `crossCutting` checks without being organized by them: branch protection and the ruleset diffs, secret names, Dependabot ecosystems, the cspell single source, section presence, script file names and secret-file stems, and `driftNotes` freshness. So read a clean run precisely. It is evidence for that subset, it is **no** evidence for any of a type's checks, and it is partial evidence across the cross-cutting dimensions. The three are easy to conflate, because adding a check under a type changes what an auditor must judge and changes no tool's output, so the check reports nothing until someone evaluates it, and silence from a tool that was never looking reads exactly like a pass. Cite the `file:line` each check was judged against, since that citation is the only durable record that the judgment happened. +**Every check under a project type is judged by hand. The cross-cutting dimensions are only partly mechanized, and the line between the two halves is not where a reader assumes.** [`spec/audit.py`][audit-runner] evaluates **no** check belonging to a type in `spec/project-types.json`, and it reads that file for one purpose only, to resolve the id a registry `driftNote` names (section 8) against the catalog and against the repo's declared types. The runner's own checks, listed in `spec/checks.json`, resolve an id too. Resolving an id is not running the check it names. What the runner does mechanize is the deterministic subset in section 0, and that subset lands on several `crossCutting` checks without being organized by them: branch protection and the ruleset diffs, secret names, Dependabot ecosystems, the cspell single source, section presence, script file names and secret-file stems, and `driftNotes` freshness. So read a clean run precisely. It is evidence for that subset, it is **no** evidence for any of a type's checks, and it is partial evidence across the cross-cutting dimensions. The three are easy to conflate, because adding a check under a type changes what an auditor must judge and changes no tool's output, so the check reports nothing until someone evaluates it, and silence from a tool that was never looking reads exactly like a pass. Cite the `file:line` each check was judged against, since that citation is the only durable record that the judgment happened. - **letter** - the exact file, section, config, or construct is present. - **intent** - an equivalent outcome holds even if the form differs. @@ -203,7 +203,7 @@ A downstream repo uses its context where it is worth most. It **files findings a **Re-running the audit needs a full hub clone with git history.** The verbatim stale-vs-modified classification and the intent staleness advisory walk the canonical's history (`git log` / `git show` from the hub root). A shallow hub clone cannot answer "matches a past hub revision". In one, `spec/audit.py` reports a single ERROR per repo and exits non-zero. `spec/audit.py --issue` exits 2 with the error on stderr. `spec/fidelity_honesty.py` stops before writing a report. Each error says to run `git fetch --unshallow origin` in that checkout. A downstream agent verifying one finding without the full history can instead compare against the current hub canonical on `main` (the whole file for a file-level unit, or the named `## heading` block for a verbatim section), which decides current-match but not stale-vs-modified. An agent picking up such an issue **re-runs the audit first and acts on the live result, not the pasted findings**: a repo moves between filing and pickup, so a stale block leads an agent to "fix" what is already fixed (re-requesting secrets that exist, attempting a no-op forward-sync). State the findings as evidence for *why* the issue was filed, never as the current state. -**Reconcile `driftNotes` in the same pass.** A registry `driftNote` records a *current* deviation from the baseline. Once the deviation is resolved the note is deleted, not left describing finished work, since hand-maintained prose drifts silently otherwise. `spec/audit.py` flags two shapes of note, neither of them gated on the rest of the audit being clean. A note asserting outstanding work in prose ("pending", "not yet", "missing", "behind", ...) is contradicted outright by a clean audit, and where findings are open it is raised as a question of which one it means, because gating the check on a clean audit meant one standing finding a repo could not clear exempted its whole note list, and the repo carrying open findings is where a stale note is most likely. A note naming the check that would retire it, as an id in parentheses and matched with them (`(hugo.generator.pinned)`, the bare id is not detected), is the mechanically checkable shape and is surfaced on **every** run: the audit resolves the id against the catalog and confirms the repo declares its type, then hands the check itself to the auditor, since section 4 above is judged by hand. **So a note naming a check id is retired by a person, not by a run.** Write it that way anyway. The id says exactly what would close the note, and the surfaced finding puts that decision in front of whoever runs the audit rather than leaving the note to sit until someone rereads it. A note naming a suppression id is not surfaced this way, since it suppresses a finding rather than naming a check to judge. The check it suppresses reports it once stale. A note naming `(python.directories.declared)` on a repo without the `python` type suppresses the Python discovery advisory. It is stale once no `.py` file remains. A note naming `(filenaming.scripts)` exempts each path it names as a code span from the file-naming dimension. A named path the check no longer flags is stale, and so is a note naming no path. The pending-marker check above still reads either. +**Reconcile `driftNotes` in the same pass.** A registry `driftNote` records a *current* deviation from the baseline. Once the deviation is resolved the note is deleted, not left describing finished work, since hand-maintained prose drifts silently otherwise. `spec/audit.py` flags two shapes of note, neither of them gated on the rest of the audit being clean. A note asserting outstanding work in prose ("pending", "not yet", "missing", "behind", ...) is contradicted outright by a clean audit, and where findings are open it is raised as a question of which one it means, because gating the check on a clean audit meant one standing finding a repo could not clear exempted its whole note list, and the repo carrying open findings is where a stale note is most likely. A note naming the check that would retire it, as an id in parentheses and matched with them (`(hugo.generator.pinned)`, the bare id is not detected), is the mechanically checkable shape and is surfaced on **every** run: the audit resolves the id against the catalog and confirms the repo declares its type, then hands the check itself to the auditor, since section 4 above is judged by hand. An id naming a runner check in `spec/checks.json` is reported as evaluated, and the note goes once that check passes. **So a note naming a check id is retired by a person, not by a run.** Write it that way anyway. The id says exactly what would close the note, and the surfaced finding puts that decision in front of whoever runs the audit rather than leaving the note to sit until someone rereads it. A note naming a suppression id is not surfaced this way, since it suppresses a finding rather than naming a check to judge. The check it suppresses reports it once stale. A note naming `(python.directories.declared)` on a repo without the `python` type suppresses the Python discovery advisory. It is stale once no `.py` file remains. A note naming `(filenaming.scripts)` exempts each path it names as a code span from the file-naming dimension. A named path the check no longer flags is stale, and so is a note naming no path. The pending-marker check above still reads either. ## 9. Escalate diff --git a/spec/audit.py b/spec/audit.py index fad30b10d..053f8b9f4 100755 --- a/spec/audit.py +++ b/spec/audit.py @@ -48,7 +48,6 @@ undeclared_h2_headings, ) from carried_text import extract_section, normalize, strip_sections -from git_env import git_location_cleared ROOT = pathlib.Path(__file__).resolve().parent.parent @@ -1080,7 +1079,7 @@ def concurrency_caller_findings(path, text): try: doc = yaml.safe_load(text) except yaml.YAMLError: - return unreadable + return [] if not isinstance(doc, dict): return unreadable concurrency = doc.get("concurrency") @@ -2170,12 +2169,6 @@ def check_registry_driftnotes(run): return driftnote_findings(run.entry, run.spec, len(run.findings)) -def _selftest(): - """Run the self-test with no variable naming a repository in the environment.""" - with git_location_cleared(): - return _selftest_cases() - - def _selftest_cases(): """A smoke check of each helper family on constructed fixtures.""" pr = "name: T\non: pull_request\njobs:\n check:\n name: Check job\n runs-on: ubuntu-latest\n" @@ -2304,7 +2297,7 @@ def parse_args(argv=None): def main(argv=None): a = parse_args(argv) if a.selftest: - return _selftest() + return _selftest_cases() if a.branch is not None and not validate.GROUND_TRUTH_BRANCH_RE.search(a.branch): shape = validate.GROUND_TRUTH_BRANCH_SHAPE print(f"--branch {a.branch!r} does not address unencoded ({shape})", file=sys.stderr) diff --git a/spec/validate.py b/spec/validate.py index 6604314cd..2948b8739 100755 --- a/spec/validate.py +++ b/spec/validate.py @@ -160,9 +160,8 @@ def _bracket_matches(text, open_char, close_char): def contains_description_markdown_link(text): """Whether `text` carries a `[text](url)` or `[text][ref]` use, brackets/parens balanced. - Kept in sync with spec/audit.py's markdown_link_spans(), which needs the same balanced-nesting rule for - the same reason: this is a description-shaped link use inside one short string, not the carried-link - regexes above, which find a definition's target inside a whole document. + A description-shaped link use inside one short string, not the carried-link regexes above, which find a + definition's target inside a whole document. """ bracket_close = _bracket_matches(text, "[", "]") paren_close = _bracket_matches(text, "(", ")") diff --git a/spec/workflow_reuse.py b/spec/workflow_reuse.py index 7853e52b7..0b0e5e0b4 100755 --- a/spec/workflow_reuse.py +++ b/spec/workflow_reuse.py @@ -5,8 +5,8 @@ # /// """Workflow-reuse measurement: how much of the fleet's GitHub Actions YAML is a copy of a hub canonical. -Read-only and owner-run. The measurement reads the live fleet, so it is not wired into CI beyond the offline ---selftest CI runs beside the audit engine's. Reuses spec/audit.py's fleet machinery (gh, normalize, registry). +Read-only and owner-run. The measurement reads the live fleet, so CI runs only its offline --selftest. +Reuses spec/audit.py's fleet machinery (gh, normalize, registry). The hub hosts a standard workflow once, as a `workflow_call` reusable workflow a downstream repo reaches by a pinned `uses:`, and a downstream repo carries only a caller stub plus a composite-action hook for what is diff --git a/tests/test_audit_helpers.py b/tests/test_audit_helpers.py index 4129a477e..5bac17c27 100755 --- a/tests/test_audit_helpers.py +++ b/tests/test_audit_helpers.py @@ -629,6 +629,12 @@ def test_unparsed_workflow_is_one_finding(self) -> None: self.assertEqual( got, [("DRIFT", "interface: wf does not parse as YAML, so its contract is unread")] ) + publisher = ".github/workflows/publish-release.yml" + self.assertEqual(audit.concurrency_caller_findings(publisher, "jobs:\n a: [\n"), []) + entry = {"types": ["python"], "pythonDirectories": ["x"]} + self.assertEqual( + audit.python_directories_caller_findings(publisher, "jobs:\n a: [\n", entry), [] + ) CANON = "line one\nline two\nline three\n" diff --git a/tests/test_git_env.py b/tests/test_git_env.py index 1feb957b1..4dbfd57a2 100755 --- a/tests/test_git_env.py +++ b/tests/test_git_env.py @@ -57,8 +57,8 @@ def test_a_variable_set_inside_the_context_does_not_outlive_it(self) -> None: self.assertNotIn("GIT_INDEX_FILE", os.environ) -class TestAuditSelftestIgnoresAmbientGitDir(unittest.TestCase): - def test_the_selftest_leaves_the_ambient_repository_unchanged(self) -> None: +class TestAuditHelperSuiteIgnoresAmbientGitDir(unittest.TestCase): + def test_the_helper_suite_leaves_the_ambient_repository_unchanged(self) -> None: with tempfile.TemporaryDirectory() as tmp: scratch = Path(tmp) _git(scratch, "init", "-q", "--initial-branch=develop", ".") @@ -78,7 +78,17 @@ def state() -> tuple[str, str, str]: env["GIT_DIR"] = str(scratch / ".git") env["GIT_WORK_TREE"] = str(scratch) proc = subprocess.run( - [sys.executable, str(REPO / "spec" / "audit.py"), "--selftest"], + [ + sys.executable, + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_audit_helpers.py", + ], + cwd=REPO, env=env, capture_output=True, text=True, @@ -88,7 +98,6 @@ def state() -> tuple[str, str, str]: ) self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) - self.assertIn("SELFTEST PASS", proc.stdout) self.assertEqual(state(), before) diff --git a/tests/test_spec_validate.py b/tests/test_spec_validate.py index 279268885..becd63d5a 100755 --- a/tests/test_spec_validate.py +++ b/tests/test_spec_validate.py @@ -1389,7 +1389,7 @@ class GroundBranchReaderCase(unittest.TestCase): `ground_branch_of(entry, branch, None)` reads the same in source as the wired call and consults no defaults, and source text cannot tell the two apart. That gap is covered behaviorally where it can be: spec/workflow_reuse.py's --selftest resolves through its injected reader and fails if its site stops - consulting the defaults, and spec/audit.py's own ground_cases pin the function's three steps. The remaining + consulting the defaults, and tests/test_audit_helpers.py's ground cases pin the function's three steps. The remaining sites need a live GitHub API, so that gap is open at each of them and this rule does not close it: the rule refuses a reader resolving the key, which a call passing nothing useful is not. """ @@ -1406,7 +1406,6 @@ def test_no_reader_resolves_the_key_itself(self) -> None: for node in ast.walk(tree): # The function's own body is where the key is resolved rather than read through it. # A self-test is exempt beside it because it reads the key to print what it resolved, which is not a production resolve. - # In this tree that rescues four nodes, all in spec/audit.py: the two subscripts inside the function's own body, and two reads in the self-test that print what it resolved. # A fixture declaring the key as a dict-literal entry matches neither branch below, so no exemption is needed for one. if isinstance(node, ast.FunctionDef) and ( node.name == self.FUNCTION or node.name.startswith("_selftest") From 28b517c590f0b6c0afef3866232752dc7562cf9e Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 23 Oct 2026 16:19:34 -0700 Subject: [PATCH 08/20] Fix Third-Pass Review Findings on the Audit Refactor A driftNote id the catalog defines keeps its catalog reading, and only a registry-only id reads as evaluated, so the full fleet sweep now matches develop line for line. A link-free tagline is returned unchanged rather than rebuilt from tokens. An unparseable dependabot.yml is one finding rather than a missing ecosystem. The smoke self-test is gated by a test, the test module has one git identity and one temporary-directory helper, the dropped rationales are back as one line each, and OPERATIONS.md names spec/carried_text.py among the shared primitives. --- OPERATIONS.md | 1 + spec/audit.py | 41 ++++++++++++++++++++++++++++--------- spec/audit_markdown.py | 5 ++++- tests/test_audit_helpers.py | 26 +++++++++++++++++------ 4 files changed, 56 insertions(+), 17 deletions(-) diff --git a/OPERATIONS.md b/OPERATIONS.md index 37a6e8c16..b1f4583be 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -139,4 +139,5 @@ The `gh api --method PATCH repos/[owner/repo]/pulls/[number]` form still works a - [catalog/](./catalog/) holds reference snippets the audit compares implementations against. - [reports/](./reports/) holds per-repo audit output. - [scripts/](./scripts/) holds the gates that run in CI and locally, and that every fleet repository reaches rather than carries. +- `extract_section()`, `strip_sections()`, `fence_step()`, and `normalize()` in [spec/carried_text.py](./spec/carried_text.py) are the shared primitives for a carried document's `## ` regions. The audit, the carry, the skill build, and the review ledger each read through them. - `read_regular_file()` and `write_regular_file()` in [host-setup/agent-safety/claude/install.py](./host-setup/agent-safety/claude/install.py) are the shared primitives for reading and replacing a file that must be regular. Each opens without blocking and judges the open descriptor, so a FIFO or a device is refused rather than hanging the call. diff --git a/spec/audit.py b/spec/audit.py index 053f8b9f4..6ba943280 100755 --- a/spec/audit.py +++ b/spec/audit.py @@ -61,7 +61,7 @@ "allow_update_branch", "delete_branch_on_merge", ] -# The bypass_actors field is deliberately absent from this list. +# The bypass_actors field is deliberately absent, since a bypass list is a per-repository decision configure.sh leaves alone. RULESET_SUBSET = ["name", "target", "enforcement", "conditions", "rules"] # Phrases that assert work is still outstanding, matched against a registry driftNote. PENDING_MARKERS = ["pending", "not yet", "owed", "todo", "still", "behind", "missing", "absent"] @@ -583,7 +583,10 @@ def gh(path, ok404=False) -> Any: def owner_repos(owner): - """Every non-fork repository the registry owner actually owns on GitHub, paginated by hand.""" + """Every non-fork repository the registry owner actually owns on GitHub, paginated by hand. + + Run as the owner's own account, since a repository-scoped token returns an incomplete list with no error. + """ me = gh("user") login = me.get("login") if isinstance(me, dict) else None # GitHub logins are case-insensitive. @@ -657,8 +660,13 @@ def normalize_ruleset(payload): return json.dumps(sub, sort_keys=True) -def check_id_owner(spec, cid): - """`(owning project type or None, whether spec/checks.json or spec/project-types.json defines the id)`.""" +def registered_check(cid): + """The spec/checks.json entry registering `cid`, or {} where none does.""" + return next((c for c in registered_checks() if c["id"] == cid), {}) + + +def catalog_owner(spec, cid): + """`(owning project type or None, whether spec/project-types.json defines the id)`.""" types = spec.get("types") if not types: raise KeyError( @@ -668,13 +676,18 @@ def check_id_owner(spec, cid): if any(c.get("id") == cid for c in t.get("checks", [])): return name, True cross = (c for dim in types.get("crossCutting", {}).values() for c in dim.get("checks", [])) - known = any(c.get("id") == cid for c in cross) - return None, known or any(c["id"] == cid for c in registered_checks()) + return None, any(c.get("id") == cid for c in cross) + + +def check_id_owner(spec, cid): + """`(owning project type or None, whether spec/project-types.json or spec/checks.json defines the id)`.""" + owner, known = catalog_owner(spec, cid) + return owner, known or bool(registered_check(cid)) def is_suppression_id(entry, cid): """Whether a driftNote naming `cid` suppresses a registered check's finding on this repo, rather than naming a check to judge.""" - check = next((c for c in registered_checks() if c["id"] == cid), {}) + check = registered_check(cid) if "suppression" not in check: return False ending_type = check["suppression"]["endsWithType"] @@ -697,7 +710,7 @@ def driftnote_findings(entry, spec, open_count): f"registry: driftNote names check '{cid}', which neither spec/checks.json nor spec/project-types.json defines - fix the id or drop the note: {quoted}", ) ) - elif any(c["id"] == cid for c in registered_checks()): + elif not catalog_owner(spec, cid)[1]: out.append( ( "DRIFT", @@ -1126,11 +1139,11 @@ def concurrency_caller_findings(path, text): def dependabot_ecosystems(text): - """The package-ecosystem values a dependabot.yml declares, empty where it does not parse.""" + """The package-ecosystem values a dependabot.yml declares, or None where it does not parse.""" try: doc = yaml.safe_load(text) except yaml.YAMLError: - return set() + return None updates = doc.get("updates") if isinstance(doc, dict) else None if not isinstance(updates, list): return set() @@ -1289,6 +1302,7 @@ def _last_effective_change(revisions): `revisions` is newest-first (date, sha, text) triples, and an empty list returns None. The oldest revision answers where every later one is normalized churn, and a None text counts as a change. + Each entry is taken as its neighbor's parent, which holds for the hub's forward-only, squash-merged history. """ for i, (date, sha, text) in enumerate(revisions): if i + 1 == len(revisions): @@ -1813,6 +1827,13 @@ def check_dependabot_ecosystems(run): if not text: return [] declared = dependabot_ecosystems(text) + if declared is None: + return [ + ( + "DRIFT", + "dependabot: .github/dependabot.yml does not parse as YAML, so its ecosystems are unread", + ) + ] implied = {} workflows = gh(f"repos/{run.slug}/contents/.github/workflows?ref={run.ground}", ok404=True) if isinstance(workflows, list) and any( diff --git a/spec/audit_markdown.py b/spec/audit_markdown.py index 789ae4e5c..4ea697c47 100644 --- a/spec/audit_markdown.py +++ b/spec/audit_markdown.py @@ -65,8 +65,11 @@ def heading_texts(markdown): def strip_md_links(text, references=None): """`text` with each Markdown link reduced to its label, everything else kept as written.""" + tokens = _MD.parseInline(text, {"references": dict(references or {})})[0].children or [] + if not any(tok.type == "link_open" for tok in tokens): + return text out = [] - for tok in _MD.parseInline(text, {"references": dict(references or {})})[0].children or []: + for tok in tokens: if tok.type in ("text", "html_inline"): out.append(tok.content) elif tok.type == "code_inline": diff --git a/tests/test_audit_helpers.py b/tests/test_audit_helpers.py index 5bac17c27..acbf11a1b 100755 --- a/tests/test_audit_helpers.py +++ b/tests/test_audit_helpers.py @@ -649,6 +649,7 @@ def test_unparsed_workflow_is_one_finding(self) -> None: IDENTITY = ( ["git", "config", "user.email", "test@test.invalid"], ["git", "config", "user.name", "test"], + ["git", "config", "commit.gpgsign", "false"], ) @@ -1387,6 +1388,8 @@ def test_strip_md_links(self) -> None: self.assertEqual( audit_markdown.strip_md_links("Plain intro line."), "Plain intro line.", msg ) + padded = "Runs ` make ` targets with ``a `b` c``." + self.assertEqual(audit_markdown.strip_md_links(padded), padded, msg) self.assertEqual(audit_markdown.strip_md_links(nested), "See API [docs] for details.", msg) self.assertEqual(audit_markdown.strip_md_links(inner_link), "See [docs] for details.", msg) self.assertEqual( @@ -1455,14 +1458,9 @@ def test_ground_branch_of(self) -> None: self.assertEqual(audit.ground_branch_of(entry, branch, defaults), want) -class ShallowCloneCase(unittest.TestCase): +class ShallowCloneCase(GitRepoCase): """Holds the shallow-clone refusal on the history reads and on the public paths.""" - def make_dir(self) -> pathlib.Path: - tmp = tempfile.TemporaryDirectory() - self.addCleanup(tmp.cleanup) - return pathlib.Path(tmp.name) - def setUp(self) -> None: clear_history_caches() self.addCleanup(clear_history_caches) @@ -4373,6 +4371,22 @@ def test_suppression_ids_resolve_against_the_registry(self) -> None: self.assertTrue(audit.is_suppression_id({"types": ["python"]}, "filenaming.scripts")) self.assertFalse(audit.is_suppression_id(typed, "settings.general")) + def test_the_smoke_selftest_passes(self) -> None: + with contextlib.redirect_stdout(io.StringIO()) as out: + self.assertEqual(audit._selftest_cases(), 0, out.getvalue()) + + def test_a_registry_only_id_reads_as_evaluated_and_a_catalog_id_does_not(self) -> None: + spec = {"types": {"types": {"python": {"checks": [{"id": "settings.general"}]}}}} + typed = {"types": ["python"], "driftNotes": ["Owed (settings.general)."]} + catalog = audit.driftnote_findings(typed, spec, 1) + self.assertNotIn("this audit evaluates", catalog[0][1]) + registry_only = audit.driftnote_findings(typed, {"types": {"types": {}}}, 1) + self.assertIn("this audit evaluates (spec/checks.json)", registry_only[0][1]) + + def test_an_unparsed_dependabot_file_reads_as_unparsed(self) -> None: + self.assertIsNone(audit.dependabot_ecosystems("updates:\n - [\n")) + self.assertEqual(audit.dependabot_ecosystems("updates: []\n"), set()) + def test_a_registry_id_is_a_known_check_id(self) -> None: spec: dict[str, Any] = {"types": {"types": {}, "crossCutting": {}}} self.assertEqual(audit.check_id_owner(spec, "settings.general"), (None, True)) From ac60822bfe4d755a8e452b82e7f7626a29b23ebf Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 23 Oct 2026 16:35:53 -0700 Subject: [PATCH 09/20] Run the Audit Through uv Wherever a Line Invokes It Each line that invokes spec/audit.py, the generated issue body, the configure.sh note, AUDIT.md, RESYNC.md, and the audit-a-repo skill, now spells it uv run --no-project --script, since the script imports PyYAML. scripts/README.md and a test docstring name spec/carried_text.py's region reading, and spec/validate.py refuses an unknown check key and a ground false outside a repo check. --- .agents/skills/audit-a-repo/SKILL.md | 4 ++-- .claude-plugin/fleet-skills/.source-digests/audit-a-repo | 2 +- .claude-plugin/fleet-skills/skills/audit-a-repo/SKILL.md | 4 ++-- .github/skills/audit-a-repo/SKILL.md | 4 ++-- AUDIT.md | 2 +- RESYNC.md | 2 +- repo-config/configure.sh | 2 +- scripts/README.md | 4 ++-- spec/audit.py | 2 +- spec/validate.py | 6 ++++++ tests/test_spec_validate.py | 6 ++++++ tests/workflow_steps.py | 3 +-- 12 files changed, 26 insertions(+), 15 deletions(-) diff --git a/.agents/skills/audit-a-repo/SKILL.md b/.agents/skills/audit-a-repo/SKILL.md index 30ef27093..a15a9184c 100644 --- a/.agents/skills/audit-a-repo/SKILL.md +++ b/.agents/skills/audit-a-repo/SKILL.md @@ -14,7 +14,7 @@ The audit is the fleet's measurement procedure, and the failure shapes it guards - **Route first.** A repo with no carried instruction set, or a partial one, has a baseline that never arrived rather than drift to report, so it goes to `STANDUP.md` sections 1A and 2 first (`AUDIT.md` section 0). Auditing it anyway produces a report that is all absences and reads as catastrophe. - **Verify the host.** Run `python3 scripts/host_gate.py --repo ` from the hub checkout before any hub tool, and pass `--repo`, since a bare run skips the target's own `host-tools.json` overlay. A stale tool answers `--version`, looks healthy, and produces a wrong answer. -- **Read `main` as ground truth**, for every workflow model, and read `develop` only to detect divergence (`AUDIT.md` section 1). An `operational` repo's `develop` is mid-flight by design, so conformance work sitting there is un-promoted work, not a defect, and it counts when it reaches `main`. Use `spec/audit.py --branch ` to preview in-flight work, which stamps the override so the finding cannot be mistaken for one against ground truth. +- **Read `main` as ground truth**, for every workflow model, and read `develop` only to detect divergence (`AUDIT.md` section 1). An `operational` repo's `develop` is mid-flight by design, so conformance work sitting there is un-promoted work, not a defect, and it counts when it reaches `main`. Use `uv run --no-project --script spec/audit.py --branch ` to preview in-flight work, which stamps the override so the finding cannot be mistaken for one against ground truth. ## Measuring @@ -28,7 +28,7 @@ The audit is the fleet's measurement procedure, and the failure shapes it guards - **Write `reports//audit.md` from `reports/_template.md`**, findings ranked most severe first, each with the `file:line` it was judged against, and quote the run stamp, since findings are a point-in-time snapshot (`AUDIT.md` section 8). - **The hub authors the report.** A downstream repo never opens a hub pull request to write its own, which would be self-certification. Downstream context goes into issues filed against the hub instead. -- **Generate a convergence issue, never compose one**: `spec/audit.py --issue ` emits it from live findings. An agent picking such an issue up re-runs the audit first and acts on the live result, not the pasted findings. +- **Generate a convergence issue, never compose one**: `uv run --no-project --script spec/audit.py --issue ` emits it from live findings. An agent picking such an issue up re-runs the audit first and acts on the live result, not the pasted findings. - **Reconcile registry `driftNotes` in the same pass**: a resolved deviation's note is deleted, not left describing finished work, and a note naming a check id is retired by a person, not by a run (`AUDIT.md` section 8). - **Stale-versus-modified classification and the intent staleness advisory need a full hub clone with git history.** In a shallow clone the audit reports one ERROR per repo and exits non-zero, so run `git fetch --unshallow origin` in the hub checkout first. To verify one finding without history, compare against the current hub canonical on `main`, which decides current-match only. diff --git a/.claude-plugin/fleet-skills/.source-digests/audit-a-repo b/.claude-plugin/fleet-skills/.source-digests/audit-a-repo index 6c4808553..f2cecfc35 100644 --- a/.claude-plugin/fleet-skills/.source-digests/audit-a-repo +++ b/.claude-plugin/fleet-skills/.source-digests/audit-a-repo @@ -1 +1 @@ -5a9bc7db47dcc412 +e1a9ebd2d8fe8c70 diff --git a/.claude-plugin/fleet-skills/skills/audit-a-repo/SKILL.md b/.claude-plugin/fleet-skills/skills/audit-a-repo/SKILL.md index 30ef27093..a15a9184c 100644 --- a/.claude-plugin/fleet-skills/skills/audit-a-repo/SKILL.md +++ b/.claude-plugin/fleet-skills/skills/audit-a-repo/SKILL.md @@ -14,7 +14,7 @@ The audit is the fleet's measurement procedure, and the failure shapes it guards - **Route first.** A repo with no carried instruction set, or a partial one, has a baseline that never arrived rather than drift to report, so it goes to `STANDUP.md` sections 1A and 2 first (`AUDIT.md` section 0). Auditing it anyway produces a report that is all absences and reads as catastrophe. - **Verify the host.** Run `python3 scripts/host_gate.py --repo ` from the hub checkout before any hub tool, and pass `--repo`, since a bare run skips the target's own `host-tools.json` overlay. A stale tool answers `--version`, looks healthy, and produces a wrong answer. -- **Read `main` as ground truth**, for every workflow model, and read `develop` only to detect divergence (`AUDIT.md` section 1). An `operational` repo's `develop` is mid-flight by design, so conformance work sitting there is un-promoted work, not a defect, and it counts when it reaches `main`. Use `spec/audit.py --branch ` to preview in-flight work, which stamps the override so the finding cannot be mistaken for one against ground truth. +- **Read `main` as ground truth**, for every workflow model, and read `develop` only to detect divergence (`AUDIT.md` section 1). An `operational` repo's `develop` is mid-flight by design, so conformance work sitting there is un-promoted work, not a defect, and it counts when it reaches `main`. Use `uv run --no-project --script spec/audit.py --branch ` to preview in-flight work, which stamps the override so the finding cannot be mistaken for one against ground truth. ## Measuring @@ -28,7 +28,7 @@ The audit is the fleet's measurement procedure, and the failure shapes it guards - **Write `reports//audit.md` from `reports/_template.md`**, findings ranked most severe first, each with the `file:line` it was judged against, and quote the run stamp, since findings are a point-in-time snapshot (`AUDIT.md` section 8). - **The hub authors the report.** A downstream repo never opens a hub pull request to write its own, which would be self-certification. Downstream context goes into issues filed against the hub instead. -- **Generate a convergence issue, never compose one**: `spec/audit.py --issue ` emits it from live findings. An agent picking such an issue up re-runs the audit first and acts on the live result, not the pasted findings. +- **Generate a convergence issue, never compose one**: `uv run --no-project --script spec/audit.py --issue ` emits it from live findings. An agent picking such an issue up re-runs the audit first and acts on the live result, not the pasted findings. - **Reconcile registry `driftNotes` in the same pass**: a resolved deviation's note is deleted, not left describing finished work, and a note naming a check id is retired by a person, not by a run (`AUDIT.md` section 8). - **Stale-versus-modified classification and the intent staleness advisory need a full hub clone with git history.** In a shallow clone the audit reports one ERROR per repo and exits non-zero, so run `git fetch --unshallow origin` in the hub checkout first. To verify one finding without history, compare against the current hub canonical on `main`, which decides current-match only. diff --git a/.github/skills/audit-a-repo/SKILL.md b/.github/skills/audit-a-repo/SKILL.md index 30ef27093..a15a9184c 100644 --- a/.github/skills/audit-a-repo/SKILL.md +++ b/.github/skills/audit-a-repo/SKILL.md @@ -14,7 +14,7 @@ The audit is the fleet's measurement procedure, and the failure shapes it guards - **Route first.** A repo with no carried instruction set, or a partial one, has a baseline that never arrived rather than drift to report, so it goes to `STANDUP.md` sections 1A and 2 first (`AUDIT.md` section 0). Auditing it anyway produces a report that is all absences and reads as catastrophe. - **Verify the host.** Run `python3 scripts/host_gate.py --repo ` from the hub checkout before any hub tool, and pass `--repo`, since a bare run skips the target's own `host-tools.json` overlay. A stale tool answers `--version`, looks healthy, and produces a wrong answer. -- **Read `main` as ground truth**, for every workflow model, and read `develop` only to detect divergence (`AUDIT.md` section 1). An `operational` repo's `develop` is mid-flight by design, so conformance work sitting there is un-promoted work, not a defect, and it counts when it reaches `main`. Use `spec/audit.py --branch ` to preview in-flight work, which stamps the override so the finding cannot be mistaken for one against ground truth. +- **Read `main` as ground truth**, for every workflow model, and read `develop` only to detect divergence (`AUDIT.md` section 1). An `operational` repo's `develop` is mid-flight by design, so conformance work sitting there is un-promoted work, not a defect, and it counts when it reaches `main`. Use `uv run --no-project --script spec/audit.py --branch ` to preview in-flight work, which stamps the override so the finding cannot be mistaken for one against ground truth. ## Measuring @@ -28,7 +28,7 @@ The audit is the fleet's measurement procedure, and the failure shapes it guards - **Write `reports//audit.md` from `reports/_template.md`**, findings ranked most severe first, each with the `file:line` it was judged against, and quote the run stamp, since findings are a point-in-time snapshot (`AUDIT.md` section 8). - **The hub authors the report.** A downstream repo never opens a hub pull request to write its own, which would be self-certification. Downstream context goes into issues filed against the hub instead. -- **Generate a convergence issue, never compose one**: `spec/audit.py --issue ` emits it from live findings. An agent picking such an issue up re-runs the audit first and acts on the live result, not the pasted findings. +- **Generate a convergence issue, never compose one**: `uv run --no-project --script spec/audit.py --issue ` emits it from live findings. An agent picking such an issue up re-runs the audit first and acts on the live result, not the pasted findings. - **Reconcile registry `driftNotes` in the same pass**: a resolved deviation's note is deleted, not left describing finished work, and a note naming a check id is retired by a person, not by a run (`AUDIT.md` section 8). - **Stale-versus-modified classification and the intent staleness advisory need a full hub clone with git history.** In a shallow clone the audit reports one ERROR per repo and exits non-zero, so run `git fetch --unshallow origin` in the hub checkout first. To verify one finding without history, compare against the current hub canonical on `main`, which decides current-match only. diff --git a/AUDIT.md b/AUDIT.md index eb1113980..ed5a2add8 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -199,7 +199,7 @@ A downstream repo uses its context where it is worth most. It **files findings a **Findings are a point-in-time snapshot. Stamp them and re-verify before acting.** [`spec/audit.py`][audit-runner] prints a run stamp (`audit run | hub `) and, per repo, the exact commit it read (`@ @`). Anything derived from a run (a report, and especially an **onboarding or conformance issue**) quotes that stamp, so a reader can tell whether it still applies. A convergence issue is generated from the audit, never composed by hand: `spec/audit.py --issue ` emits a ready-to-file title and body from that repo's live findings (grouped into must-fix, converge, and could-not-verify), so the issue content cannot drift from what the audit actually found and regenerates as the repo changes. -**Verify a convergence before it is promoted with `--branch`.** `spec/audit.py --branch ` reads that ref instead of the repo's registry `groundTruthBranch`, so a repo can audit its own `develop` while the work is still in flight rather than discovering the gaps after `main` has moved. The registry is not edited, the run is still read-only, and the run stamp names the override so a finding cannot be mistaken for one against ground truth. A ref that does not resolve is a single error naming it, never a baseline's worth of file-absent letters. A ref off the grammar a registry `groundTruthBranch` is held to, `GROUND_TRUTH_BRANCH_PATTERN` in [`spec/validate.py`][validate], is refused before anything is read. The run then exits `2` with one line on stderr naming the ref. +**Verify a convergence before it is promoted with `--branch`.** `uv run --no-project --script spec/audit.py --branch ` reads that ref instead of the repo's registry `groundTruthBranch`, so a repo can audit its own `develop` while the work is still in flight rather than discovering the gaps after `main` has moved. The registry is not edited, the run is still read-only, and the run stamp names the override so a finding cannot be mistaken for one against ground truth. A ref that does not resolve is a single error naming it, never a baseline's worth of file-absent letters. A ref off the grammar a registry `groundTruthBranch` is held to, `GROUND_TRUTH_BRANCH_PATTERN` in [`spec/validate.py`][validate], is refused before anything is read. The run then exits `2` with one line on stderr naming the ref. **Re-running the audit needs a full hub clone with git history.** The verbatim stale-vs-modified classification and the intent staleness advisory walk the canonical's history (`git log` / `git show` from the hub root). A shallow hub clone cannot answer "matches a past hub revision". In one, `spec/audit.py` reports a single ERROR per repo and exits non-zero. `spec/audit.py --issue` exits 2 with the error on stderr. `spec/fidelity_honesty.py` stops before writing a report. Each error says to run `git fetch --unshallow origin` in that checkout. A downstream agent verifying one finding without the full history can instead compare against the current hub canonical on `main` (the whole file for a file-level unit, or the named `## heading` block for a verbatim section), which decides current-match but not stale-vs-modified. An agent picking up such an issue **re-runs the audit first and acts on the live result, not the pasted findings**: a repo moves between filing and pickup, so a stale block leads an agent to "fix" what is already fixed (re-requesting secrets that exist, attempting a no-op forward-sync). State the findings as evidence for *why* the issue was filed, never as the current state. diff --git a/RESYNC.md b/RESYNC.md index 1667290ad..220232554 100644 --- a/RESYNC.md +++ b/RESYNC.md @@ -26,7 +26,7 @@ Three states look similar from inside a repository and take different procedures - **A partial instruction set.** [`STANDUP.md`][standup] sections 1A and 2. An absent carried file is a baseline that never arrived, not drift, and the two are fixed differently: a baseline is carried, and drift is converged. Resyncing a repository whose `AGENTS.md` never carried the rule sections would re-vendor sections into a document that was never given them, which reads as drift repair and is really a late standup. - **A full instruction set, current or stale.** This file. Stale is the normal case and is not a defect, since the hub moves and a repository is a snapshot of the revision it last converged with. -**The distinction is measured, not assumed.** `spec/audit.py ` reports an absent carried file as a `LETTER` and a present-but-stale one as a `DRIFT`, so the finding kinds themselves say which procedure the repository is owed. A run that is mostly letters is a repository that needs [`STANDUP.md`][standup]. +**The distinction is measured, not assumed.** `uv run --no-project --script spec/audit.py ` reports an absent carried file as a `LETTER` and a present-but-stale one as a `DRIFT`, so the finding kinds themselves say which procedure the repository is owed. A run that is mostly letters is a repository that needs [`STANDUP.md`][standup]. **A few letters are not that, and the ordinary cause is the baseline having grown.** When the manifest gains an entry, every repository in the fleet reports that one file absent on its next audit, all at once and through no fault of its own. That is a file to carry inside a resync, at the point in section 3 where its class belongs, rather than evidence the repository was never stood up. Read the letters before routing on their count: a handful naming files the hub added recently is a resync, and a baseline's worth naming the instruction set is a standup. diff --git a/repo-config/configure.sh b/repo-config/configure.sh index cffd37b70..aa4b72eee 100755 --- a/repo-config/configure.sh +++ b/repo-config/configure.sh @@ -791,7 +791,7 @@ cmd_check() { check_environments # Secret names are asserted by spec/audit.py, not here. # A secret's value is never readable via the API regardless, repository-scoped and environment-scoped alike. - note "run spec/audit.py [RepoName] (the registry name, not owner/repo) for required secret names, then verify by hand that their values are valid" + note "run uv run --no-project --script spec/audit.py [RepoName] (the registry name, not owner/repo) for required secret names, then verify by hand that their values are valid" if [ "$FAILED" -ne 0 ]; then echo "Configuration drift detected on $repo." exit 1 diff --git a/scripts/README.md b/scripts/README.md index 49e376c7a..1a16dc1d0 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -376,11 +376,11 @@ The tool validates the registry identity, target origin, feature-branch worktree The section modes close the gap between the two things that already existed: a tree mode owns a whole tree, and [`spec/audit.py`][audit] names a stale section precisely without writing anything, so the one apply step in [`RESYNC.md`][resync] with no tested tool behind it was the step the AGENTS.md-overwrite incident happened on. `check-sections` reports each declared verbatim section and which of them are stale, and exits `1` while any is. `apply-sections` replaces only those regions and then re-reads the file to assert three things: the level-two heading sequence is unchanged, every line outside the replaced regions is byte-identical to what was there before, and each declared section now matches the hub's canonical. The second of those is the one that earns its keep, since the defect it exists for, one blank line dropped between the preamble and the first heading, is invisible in a diff review, breaks no renderer, and no gate in this repository flags it. -Region boundaries come from `spec/audit.py`'s own fence reading rather than a second reading of CommonMark, so a level-two heading marker shown inside a code sample cannot end a region for this tool and not for the check that judges the result. The replacement carries the terminator the section's own heading line carried, so a CRLF repository keeps its endings and the file is never rewritten to one ending it may not use throughout. +Region boundaries come from `spec/carried_text.py`'s fence reading, which the audit shares, rather than a second reading of CommonMark, so a level-two heading marker shown inside a code sample cannot end a region for this tool and not for the check that judges the result. The replacement carries the terminator the section's own heading line carried, so a CRLF repository keeps its endings and the file is never rewritten to one ending it may not use throughout. Every refusal is raised while planning, before the first file is written, so a refused run leaves nothing re-vendored. What refuses: a section the manifest declares that the repository does not carry (where it belongs is a standup question), a file absent from the target, a file that is not UTF-8 or that carries a bare carriage return, which the fidelity comparison reads as a line break and this tool does not, a file declaring both placeholders and verbatim sections (a verbatim unit carries none, per [`spec/fidelity-model.md`][fidelity-model]), a region holding content the fidelity comparison normalizes away, such as a Dependabot-owned action pin, whose bytes the repository rather than the hub owns, and two declared names resolving to one region. A failure after that point is the write itself or its assertion, and it names the file it was working on, because an earlier file may already hold its re-vendored content. -A region runs to the next level-two heading, so the blank line before that heading sits inside the region and a section ending its file has none. `spec/audit.py`'s `extract_section` reads those two as the same content, since it splits the document on LF and joins the region back with LF, so a file-final newline and a blank line before a following heading both render as one trailing empty element. The comparison therefore uses that form, and the write reconciles the boundary to what the target document needs rather than copying it from the hub: a section that ends the hub's file keeps a blank line before whatever follows it downstream, and one that ends the target's file gains no trailing blank line. +A region runs to the next level-two heading, so the blank line before that heading sits inside the region and a section ending its file has none. `spec/carried_text.py`'s `extract_section` reads those two as the same content, since it splits the document on LF and joins the region back with LF, so a file-final newline and a blank line before a following heading both render as one trailing empty element. The comparison therefore uses that form, and the write reconciles the boundary to what the target document needs rather than copying it from the hub: a section that ends the hub's file keeps a blank line before whatever follows it downstream, and one that ends the target's file gains no trailing blank line. The target is required to be clean, with no owned root exempted from the unrelated-changes check, unlike a tree carry. A tree's root holds nothing but hub-owned content, so exempting it gives up nothing, where these files hold the repository's own sections too and exempting one would let a write land on top of an uncommitted edit to a section the tool never touches. diff --git a/spec/audit.py b/spec/audit.py index 6ba943280..35cec5f2e 100755 --- a/spec/audit.py +++ b/spec/audit.py @@ -2254,7 +2254,7 @@ def render_issue(entry, findings, ground, audited_sha, run_utc, hub_sha): w( f"Generated from the hub audit of `{name}` ({types}). Run stamp `audit run {run_utc} | hub {hub_sha}`, " f"against `@ {stamp}` (the format AUDIT.md section 8 says a derived artifact quotes). Regenerate with " - f"`spec/audit.py --issue {name}`. Findings are a point-in-time snapshot - re-run the audit before acting. " + f"`uv run --no-project --script spec/audit.py --issue {name}`. Findings are a point-in-time snapshot - re-run the audit before acting. " f"This lists what the audit mechanically detects. No check belonging to a project type in `spec/project-types.json` is run " f"here, and the cross-cutting dimensions are covered only in part, so the full letter and intent verdict lives in AUDIT.md section 4." ) diff --git a/spec/validate.py b/spec/validate.py index 2948b8739..a7256a2f9 100755 --- a/spec/validate.py +++ b/spec/validate.py @@ -120,6 +120,7 @@ # A driftNote names a check as a parenthesized id, "(hugo.generator.pinned)", anywhere in the note. CHECK_ID_RE = re.compile(r"\(([a-z][a-z0-9-]*(?:\.[a-z0-9-]+)+)\)") CHECK_SCOPES = ("repo", "baseline", "file") +CHECK_KEYS = {"id", "family", "description", "spec", "hub", "scope", "ground", "suppression"} MARKDOWN_INLINE_LINK = re.compile(r"\]\((?P[^)\s]+)") MARKDOWN_REFERENCE_LINK = re.compile(r"^\[[^]]+\]:\s*(?P\S+)", re.MULTILINE) TEMPLATE_REPOSITORY_URL = "https://github.com/ptr727/ProjectTemplate" @@ -423,6 +424,11 @@ def check_registry_errors(registry, types, repos): errors.append(f"{where} scope must be one of {', '.join(CHECK_SCOPES)}") if not isinstance(check.get("ground", True), bool): errors.append(f"{where} ground must be true or false") + elif check.get("ground") is False and check.get("scope") != "repo": + errors.append(f"{where} ground false applies only to a repo check") + unknown = sorted(set(check) - CHECK_KEYS) + if unknown: + errors.append(f"{where} carries unknown key(s) {', '.join(unknown)}") suppression = check.get("suppression") ending = suppression.get("endsWithType", 0) if isinstance(suppression, dict) else 0 if "suppression" in check and not (ending is None or ending in types.get("types", {})): diff --git a/tests/test_spec_validate.py b/tests/test_spec_validate.py index becd63d5a..f1003c174 100755 --- a/tests/test_spec_validate.py +++ b/tests/test_spec_validate.py @@ -67,6 +67,12 @@ def test_each_malformed_field_is_reported(self) -> None: ), ("a check that is no object", ["a.b"], "must be an object"), ("a file check with nothing read", [self.check(scope="file")], "no baseline check"), + ("a misspelled key", [self.check(grond=False)], "unknown key(s) grond"), + ( + "ground false on a file check", + [self.check(scope="baseline"), self.check(id="c.d", scope="file", ground=False)], + "applies only to a repo check", + ), ] for label, checks, needle in cases: with self.subTest(label): diff --git a/tests/workflow_steps.py b/tests/workflow_steps.py index b9cfa15cb..5a72bfa27 100644 --- a/tests/workflow_steps.py +++ b/tests/workflow_steps.py @@ -42,8 +42,7 @@ def step_script(path: Path, name: str) -> str: def yaml_step(text: str, name: str) -> str: """The one sequence item named `name`, its dash blanked so its keys read as a mapping. - Read by indentation, as `spec/audit.py`'s workflow readers are, since the suite runs on the - standard library alone. The item ends at the first line at or left of its dash, so a key under + Read by indentation. The item ends at the first line at or left of its dash, so a key under a sibling step is never this step's. """ lines = text.splitlines() From e8469e0a70f22dfd54558465019d60d84ca7eedf Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 23 Oct 2026 16:51:41 -0700 Subject: [PATCH 10/20] Read an Image in a Tagline as Markup, and Tidy Two Pointers A tagline carrying an image reads as carrying markup again, with the image reduced to its alt text. AUDIT.md says only an id spec/checks.json alone defines reads as evaluated, matching the catalog's precedence. The audit drops two urllib imports the Markdown split left unused, and canonical_review.py names spec/carried_text.py's normalize. --- AUDIT.md | 2 +- scripts/canonical_review.py | 2 +- spec/audit.py | 2 -- spec/audit_markdown.py | 10 +++++++--- tests/test_audit_helpers.py | 2 ++ 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/AUDIT.md b/AUDIT.md index ed5a2add8..2b3f94fe4 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -203,7 +203,7 @@ A downstream repo uses its context where it is worth most. It **files findings a **Re-running the audit needs a full hub clone with git history.** The verbatim stale-vs-modified classification and the intent staleness advisory walk the canonical's history (`git log` / `git show` from the hub root). A shallow hub clone cannot answer "matches a past hub revision". In one, `spec/audit.py` reports a single ERROR per repo and exits non-zero. `spec/audit.py --issue` exits 2 with the error on stderr. `spec/fidelity_honesty.py` stops before writing a report. Each error says to run `git fetch --unshallow origin` in that checkout. A downstream agent verifying one finding without the full history can instead compare against the current hub canonical on `main` (the whole file for a file-level unit, or the named `## heading` block for a verbatim section), which decides current-match but not stale-vs-modified. An agent picking up such an issue **re-runs the audit first and acts on the live result, not the pasted findings**: a repo moves between filing and pickup, so a stale block leads an agent to "fix" what is already fixed (re-requesting secrets that exist, attempting a no-op forward-sync). State the findings as evidence for *why* the issue was filed, never as the current state. -**Reconcile `driftNotes` in the same pass.** A registry `driftNote` records a *current* deviation from the baseline. Once the deviation is resolved the note is deleted, not left describing finished work, since hand-maintained prose drifts silently otherwise. `spec/audit.py` flags two shapes of note, neither of them gated on the rest of the audit being clean. A note asserting outstanding work in prose ("pending", "not yet", "missing", "behind", ...) is contradicted outright by a clean audit, and where findings are open it is raised as a question of which one it means, because gating the check on a clean audit meant one standing finding a repo could not clear exempted its whole note list, and the repo carrying open findings is where a stale note is most likely. A note naming the check that would retire it, as an id in parentheses and matched with them (`(hugo.generator.pinned)`, the bare id is not detected), is the mechanically checkable shape and is surfaced on **every** run: the audit resolves the id against the catalog and confirms the repo declares its type, then hands the check itself to the auditor, since section 4 above is judged by hand. An id naming a runner check in `spec/checks.json` is reported as evaluated, and the note goes once that check passes. **So a note naming a check id is retired by a person, not by a run.** Write it that way anyway. The id says exactly what would close the note, and the surfaced finding puts that decision in front of whoever runs the audit rather than leaving the note to sit until someone rereads it. A note naming a suppression id is not surfaced this way, since it suppresses a finding rather than naming a check to judge. The check it suppresses reports it once stale. A note naming `(python.directories.declared)` on a repo without the `python` type suppresses the Python discovery advisory. It is stale once no `.py` file remains. A note naming `(filenaming.scripts)` exempts each path it names as a code span from the file-naming dimension. A named path the check no longer flags is stale, and so is a note naming no path. The pending-marker check above still reads either. +**Reconcile `driftNotes` in the same pass.** A registry `driftNote` records a *current* deviation from the baseline. Once the deviation is resolved the note is deleted, not left describing finished work, since hand-maintained prose drifts silently otherwise. `spec/audit.py` flags two shapes of note, neither of them gated on the rest of the audit being clean. A note asserting outstanding work in prose ("pending", "not yet", "missing", "behind", ...) is contradicted outright by a clean audit, and where findings are open it is raised as a question of which one it means, because gating the check on a clean audit meant one standing finding a repo could not clear exempted its whole note list, and the repo carrying open findings is where a stale note is most likely. A note naming the check that would retire it, as an id in parentheses and matched with them (`(hugo.generator.pinned)`, the bare id is not detected), is the mechanically checkable shape and is surfaced on **every** run: the audit resolves the id against the catalog and confirms the repo declares its type, then hands the check itself to the auditor, since section 4 above is judged by hand. An id only `spec/checks.json` defines is reported as evaluated, and the note goes once that check passes. **So a note naming a check id is retired by a person, not by a run.** Write it that way anyway. The id says exactly what would close the note, and the surfaced finding puts that decision in front of whoever runs the audit rather than leaving the note to sit until someone rereads it. A note naming a suppression id is not surfaced this way, since it suppresses a finding rather than naming a check to judge. The check it suppresses reports it once stale. A note naming `(python.directories.declared)` on a repo without the `python` type suppresses the Python discovery advisory. It is stale once no `.py` file remains. A note naming `(filenaming.scripts)` exempts each path it names as a code span from the file-naming dimension. A named path the check no longer flags is stale, and so is a note naming no path. The pending-marker check above still reads either. ## 9. Escalate diff --git a/scripts/canonical_review.py b/scripts/canonical_review.py index e9335310b..1a911339b 100755 --- a/scripts/canonical_review.py +++ b/scripts/canonical_review.py @@ -123,7 +123,7 @@ def normalize(text: str) -> str: """A unit's comparable form, which neutralizes line endings and nothing else. - Deliberately not `spec/audit.py`'s `normalize`, which also masks Dependabot-owned action pins + Deliberately not `spec/carried_text.py`'s `normalize`, which also masks Dependabot-owned action pins and per-repo `needs:` lists. Those are governed drift for a fidelity comparison, where the question is whether two copies say the same thing. The question here is whether a reviewer has read these bytes, and masking any of them would report a read of text nobody saw. Line endings diff --git a/spec/audit.py b/spec/audit.py index 35cec5f2e..084412c5d 100755 --- a/spec/audit.py +++ b/spec/audit.py @@ -26,9 +26,7 @@ import re import subprocess import sys -import urllib.error import urllib.parse -import urllib.request from dataclasses import dataclass, field from datetime import UTC, datetime from typing import Any diff --git a/spec/audit_markdown.py b/spec/audit_markdown.py index 4ea697c47..656518ecf 100644 --- a/spec/audit_markdown.py +++ b/spec/audit_markdown.py @@ -64,13 +64,17 @@ def heading_texts(markdown): def strip_md_links(text, references=None): - """`text` with each Markdown link reduced to its label, everything else kept as written.""" + """`text` with each Markdown link reduced to its label and each image to its alt text. + + `references` is the document's reference table, so `[text][ref]` is a link only where `ref` is defined. + A text carrying neither comes back unchanged. + """ tokens = _MD.parseInline(text, {"references": dict(references or {})})[0].children or [] - if not any(tok.type == "link_open" for tok in tokens): + if not any(tok.type in ("link_open", "image") for tok in tokens): return text out = [] for tok in tokens: - if tok.type in ("text", "html_inline"): + if tok.type in ("text", "html_inline", "image"): out.append(tok.content) elif tok.type == "code_inline": out.append(tok.markup + tok.content + tok.markup) diff --git a/tests/test_audit_helpers.py b/tests/test_audit_helpers.py index acbf11a1b..620d8de8e 100755 --- a/tests/test_audit_helpers.py +++ b/tests/test_audit_helpers.py @@ -1390,6 +1390,8 @@ def test_strip_md_links(self) -> None: ) padded = "Runs ` make ` targets with ``a `b` c``." self.assertEqual(audit_markdown.strip_md_links(padded), padded, msg) + imaged = "Cleans media ![ci](https://img.example/b.svg) fast." + self.assertEqual(audit_markdown.strip_md_links(imaged), "Cleans media ci fast.", msg) self.assertEqual(audit_markdown.strip_md_links(nested), "See API [docs] for details.", msg) self.assertEqual(audit_markdown.strip_md_links(inner_link), "See [docs] for details.", msg) self.assertEqual( From d8698afdea87bc95416abf51002731ba9d80a540 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 23 Oct 2026 17:09:47 -0700 Subject: [PATCH 11/20] Fix Review Findings on the Registry Validator and Doc Lines spec/validate.py reports a malformed check id or suppression type rather than raising on it, and holds the baseline-before-file rule on the hub's filtered run too. The remaining invocation lines run the audit and the fidelity report through uv, AUDIT.md's id-resolution sentence says what the runner does, the test module derives its init sequence from one identity tuple, and OPERATIONS.md drops a code span markdownlint refuses. --- AUDIT.md | 4 ++-- OPERATIONS.md | 2 +- README.md | 2 +- docs/repo-config.md | 2 +- spec/validate.py | 25 ++++++++++++++++++------- tests/test_audit_helpers.py | 7 +------ tests/test_spec_validate.py | 11 +++++++++++ 7 files changed, 35 insertions(+), 18 deletions(-) diff --git a/AUDIT.md b/AUDIT.md index 2b3f94fe4..406f763cb 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -69,7 +69,7 @@ Which carried files and sections a repo is expected to have is decided by its sc For each applicable type in [`spec/project-types.json`][project-types] and every cross-cutting dimension, evaluate each check at its stated verdict tier: -**Every check under a project type is judged by hand. The cross-cutting dimensions are only partly mechanized, and the line between the two halves is not where a reader assumes.** [`spec/audit.py`][audit-runner] evaluates **no** check belonging to a type in `spec/project-types.json`, and it reads that file for one purpose only, to resolve the id a registry `driftNote` names (section 8) against the catalog and against the repo's declared types. The runner's own checks, listed in `spec/checks.json`, resolve an id too. Resolving an id is not running the check it names. What the runner does mechanize is the deterministic subset in section 0, and that subset lands on several `crossCutting` checks without being organized by them: branch protection and the ruleset diffs, secret names, Dependabot ecosystems, the cspell single source, section presence, script file names and secret-file stems, and `driftNotes` freshness. So read a clean run precisely. It is evidence for that subset, it is **no** evidence for any of a type's checks, and it is partial evidence across the cross-cutting dimensions. The three are easy to conflate, because adding a check under a type changes what an auditor must judge and changes no tool's output, so the check reports nothing until someone evaluates it, and silence from a tool that was never looking reads exactly like a pass. Cite the `file:line` each check was judged against, since that citation is the only durable record that the judgment happened. +**Every check under a project type is judged by hand. The cross-cutting dimensions are only partly mechanized, and the line between the two halves is not where a reader assumes.** [`spec/audit.py`][audit-runner] evaluates **no** check belonging to a type in `spec/project-types.json`, and it reads that file for one purpose only, to resolve the id a registry `driftNote` names (section 8) against the catalog and against the repo's declared types. An id is resolved against the runner's own checks in `spec/checks.json` too. Resolving a catalog id is not running the check it names. What the runner does mechanize is the deterministic subset in section 0, and that subset lands on several `crossCutting` checks without being organized by them: branch protection and the ruleset diffs, secret names, Dependabot ecosystems, the cspell single source, section presence, script file names and secret-file stems, and `driftNotes` freshness. So read a clean run precisely. It is evidence for that subset, it is **no** evidence for any of a type's checks, and it is partial evidence across the cross-cutting dimensions. The three are easy to conflate, because adding a check under a type changes what an auditor must judge and changes no tool's output, so the check reports nothing until someone evaluates it, and silence from a tool that was never looking reads exactly like a pass. Cite the `file:line` each check was judged against, since that citation is the only durable record that the judgment happened. - **letter** - the exact file, section, config, or construct is present. - **intent** - an equivalent outcome holds even if the form differs. @@ -197,7 +197,7 @@ Write `reports//audit.md` from [`reports/_template.md`][template]: a dimen A downstream repo uses its context where it is worth most. It **files findings about the hub as issues against `ptr727/ProjectTemplate`** and **applies fixes to its own repo** per section 10. Filing an issue is the opposite of self-certification, and it is how several hub defects have been found. Hub findings include bugs, conflicting sources, unclear or incomplete instructions, missing capabilities, and Copilot findings about any of them. Search open and closed issues first, then update the matching issue or file a new one. Any pull request it opens against the hub follows the standard branching model. It targets `develop`, never `main`. -**Findings are a point-in-time snapshot. Stamp them and re-verify before acting.** [`spec/audit.py`][audit-runner] prints a run stamp (`audit run | hub `) and, per repo, the exact commit it read (`@ @`). Anything derived from a run (a report, and especially an **onboarding or conformance issue**) quotes that stamp, so a reader can tell whether it still applies. A convergence issue is generated from the audit, never composed by hand: `spec/audit.py --issue ` emits a ready-to-file title and body from that repo's live findings (grouped into must-fix, converge, and could-not-verify), so the issue content cannot drift from what the audit actually found and regenerates as the repo changes. +**Findings are a point-in-time snapshot. Stamp them and re-verify before acting.** [`spec/audit.py`][audit-runner] prints a run stamp (`audit run | hub `) and, per repo, the exact commit it read (`@ @`). Anything derived from a run (a report, and especially an **onboarding or conformance issue**) quotes that stamp, so a reader can tell whether it still applies. A convergence issue is generated from the audit, never composed by hand: `uv run --no-project --script spec/audit.py --issue ` emits a ready-to-file title and body from that repo's live findings (grouped into must-fix, converge, and could-not-verify), so the issue content cannot drift from what the audit actually found and regenerates as the repo changes. **Verify a convergence before it is promoted with `--branch`.** `uv run --no-project --script spec/audit.py --branch ` reads that ref instead of the repo's registry `groundTruthBranch`, so a repo can audit its own `develop` while the work is still in flight rather than discovering the gaps after `main` has moved. The registry is not edited, the run is still read-only, and the run stamp names the override so a finding cannot be mistaken for one against ground truth. A ref that does not resolve is a single error naming it, never a baseline's worth of file-absent letters. A ref off the grammar a registry `groundTruthBranch` is held to, `GROUND_TRUTH_BRANCH_PATTERN` in [`spec/validate.py`][validate], is refused before anything is read. The run then exits `2` with one line on stderr naming the ref. diff --git a/OPERATIONS.md b/OPERATIONS.md index b1f4583be..fc73cb931 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -139,5 +139,5 @@ The `gh api --method PATCH repos/[owner/repo]/pulls/[number]` form still works a - [catalog/](./catalog/) holds reference snippets the audit compares implementations against. - [reports/](./reports/) holds per-repo audit output. - [scripts/](./scripts/) holds the gates that run in CI and locally, and that every fleet repository reaches rather than carries. -- `extract_section()`, `strip_sections()`, `fence_step()`, and `normalize()` in [spec/carried_text.py](./spec/carried_text.py) are the shared primitives for a carried document's `## ` regions. The audit, the carry, the skill build, and the review ledger each read through them. +- `extract_section()`, `strip_sections()`, `fence_step()`, and `normalize()` in [spec/carried_text.py](./spec/carried_text.py) are the shared primitives for a carried document's level-two heading regions. The audit, the carry, the skill build, and the review ledger each read through them. - `read_regular_file()` and `write_regular_file()` in [host-setup/agent-safety/claude/install.py](./host-setup/agent-safety/claude/install.py) are the shared primitives for reading and replacing a file that must be regular. Each opens without blocking and judges the open descriptor, so a FIFO or a device is refused rather than hanging the call. diff --git a/README.md b/README.md index cdee53076..a7ae17657 100644 --- a/README.md +++ b/README.md @@ -215,7 +215,7 @@ A rule that cannot be diverged from is a rule people work around silently, which **Check the unit's fidelity level first, because most apparent divergences are not divergences at all.** Each [spec/files.json][files] entry declares one of `verbatim`, `interface`, `intent`, or `presence`, defaulting to `presence`, and the two permissive levels already hand the repo its content outright. A repo rewriting a `presence` or `intent` file to suit itself is exercising the freedom the level grants rather than breaking a rule, and the audit asserts nothing beyond presence for it. See [spec/fidelity-model.md][fidelity-model] for which unit sits where and why. -**A divergence from a `verbatim` or `interface` unit is recorded, not hidden.** The ledger is [spec/divergences.json][divergences]. Every entry names a path, a disposition, and a reason. A `dispositions` entry also names the repos it covers. An `investigate` entry carries a `tracking` value naming where its pending decision is being made, and every other disposition may leave that value null. `spec/fidelity_honesty.py --report` joins the ledger against live fleet reality to regenerate [reports/divergences.md][divergences-report]. A divergence recorded as `accepted` is a legitimate permanent one and no further action is owed. A live divergence with no entry renders as `UNTRIAGED`, which is the point: the audit does not care whether a difference is deliberate, only whether someone decided about it. Edit the ledger and regenerate the report, never the report. +**A divergence from a `verbatim` or `interface` unit is recorded, not hidden.** The ledger is [spec/divergences.json][divergences]. Every entry names a path, a disposition, and a reason. A `dispositions` entry also names the repos it covers. An `investigate` entry carries a `tracking` value naming where its pending decision is being made, and every other disposition may leave that value null. `uv run --no-project --script spec/fidelity_honesty.py --report` joins the ledger against live fleet reality to regenerate [reports/divergences.md][divergences-report]. A divergence recorded as `accepted` is a legitimate permanent one and no further action is owed. A live divergence with no entry renders as `UNTRIAGED`, which is the point: the audit does not care whether a difference is deliberate, only whether someone decided about it. Edit the ledger and regenerate the report, never the report. **A rule that is wrong for many repositories is a hub defect rather than a per-repo exception.** [AUDIT.md][audit] section 9 says so directly, that a repeated letter miss many repos share is a signal the spec needs adjusting, so raise it here instead of accumulating one exception per repository. diff --git a/docs/repo-config.md b/docs/repo-config.md index 0177df2af..266b90057 100644 --- a/docs/repo-config.md +++ b/docs/repo-config.md @@ -20,7 +20,7 @@ The carried `AUDIT.md` reaches the hub at `main` for its configuration check. Th ## Per-Repository Secrets -Downstream repositories carry no copy of `spec/secrets.json`. `baseline` applies to every fleet repo by definition, and `mechanisms`/`targetMechanisms`/`typeMechanisms` resolve from the registry entry (a repo's `publish[]` targets and `types[]`) rather than from anything repo-specific, so a per-repo copy could only restate the hub's own computation or drift from it between resyncs. `spec/audit.py [RepoName]`, run from a hub checkout, resolves the required set the same way the mechanized `repo-setup` check does: `targetMechanisms`/`typeMechanisms` selects each applicable mechanism from the registry entry, `baseline` plus the selected `mechanisms` plus the registry entry's own `requiredSecrets[]` (the repo's domain-specific additions) gives the required and forbidden names, and the result is cross-checked against the repo's live Actions and Dependabot secret stores. +Downstream repositories carry no copy of `spec/secrets.json`. `baseline` applies to every fleet repo by definition, and `mechanisms`/`targetMechanisms`/`typeMechanisms` resolve from the registry entry (a repo's `publish[]` targets and `types[]`) rather than from anything repo-specific, so a per-repo copy could only restate the hub's own computation or drift from it between resyncs. `uv run --no-project --script spec/audit.py [RepoName]`, run from a hub checkout, resolves the required set the same way the mechanized `repo-setup` check does: `targetMechanisms`/`typeMechanisms` selects each applicable mechanism from the registry entry, `baseline` plus the selected `mechanisms` plus the registry entry's own `requiredSecrets[]` (the repo's domain-specific additions) gives the required and forbidden names, and the result is cross-checked against the repo's live Actions and Dependabot secret stores. ## Applying the Config diff --git a/spec/validate.py b/spec/validate.py index a7256a2f9..5ce267f54 100755 --- a/spec/validate.py +++ b/spec/validate.py @@ -396,22 +396,32 @@ def check_registry_errors(registry, types, repos): return ["checks.json: 'checks' must be an array"] errors = [] ids = set() - read = False + # A file check runs only on what a baseline check before it in its run has read, on the hub's filtered run too. + unread = {} + for hub_run in (False, True): + read = False + for i, check in enumerate(checks): + if not isinstance(check, dict) or (hub_run and check.get("hub") is not True): + continue + read = check.get("scope") == "baseline" or (read and check.get("scope") == "file") + if check.get("scope") == "file" and not read: + unread.setdefault(i, "the hub's run" if hub_run else "its run") + for i, run in sorted(unread.items()): + errors.append( + f"checks.json: checks[{i}] is a file check with no baseline check before it in {run}" + ) for i, check in enumerate(checks): where = f"checks.json: checks[{i}]" if not isinstance(check, dict): errors.append(f"{where} must be an object") continue - # A file check runs only on what a baseline check before it in the same run has read. - read = check.get("scope") == "baseline" or (read and check.get("scope") == "file") - if check.get("scope") == "file" and not read: - errors.append(f"{where} is a file check with no baseline check before it in its run") cid = check.get("id") if not isinstance(cid, str) or not CHECK_ID_RE.fullmatch(f"({cid})"): errors.append(f"{where} id {cid!r} is not a dotted lowercase check id") elif cid in ids: errors.append(f"{where} id '{cid}' is declared more than once") - ids.add(cid) + else: + ids.add(cid) for key in ("family", "description", "spec"): value = check.get(key) if not isinstance(value, str) or not value.strip() or "\n" in value: @@ -431,7 +441,8 @@ def check_registry_errors(registry, types, repos): errors.append(f"{where} carries unknown key(s) {', '.join(unknown)}") suppression = check.get("suppression") ending = suppression.get("endsWithType", 0) if isinstance(suppression, dict) else 0 - if "suppression" in check and not (ending is None or ending in types.get("types", {})): + named = isinstance(ending, str) and ending in types.get("types", {}) + if "suppression" in check and not (ending is None or named): errors.append(f"{where} suppression.endsWithType must name a project type or be null") catalog = {c.get("id") for t in types.get("types", {}).values() for c in t.get("checks", [])} catalog |= { diff --git a/tests/test_audit_helpers.py b/tests/test_audit_helpers.py index 620d8de8e..07db0b4c6 100755 --- a/tests/test_audit_helpers.py +++ b/tests/test_audit_helpers.py @@ -1012,12 +1012,7 @@ def test_another_git_failure_still_fails_loudly(self) -> None: self.assertEqual(error.cmd[:2], ["git", "commit"]) -GIT_IDENTITY = ( - ["git", "init", "-q"], - ["git", "config", "user.email", "test@test.invalid"], - ["git", "config", "user.name", "test"], - ["git", "config", "commit.gpgsign", "false"], -) +GIT_IDENTITY = (["git", "init", "-q"], *IDENTITY) def clear_history_caches() -> None: diff --git a/tests/test_spec_validate.py b/tests/test_spec_validate.py index f1003c174..274bab858 100755 --- a/tests/test_spec_validate.py +++ b/tests/test_spec_validate.py @@ -68,6 +68,17 @@ def test_each_malformed_field_is_reported(self) -> None: ("a check that is no object", ["a.b"], "must be an object"), ("a file check with nothing read", [self.check(scope="file")], "no baseline check"), ("a misspelled key", [self.check(grond=False)], "unknown key(s) grond"), + ("a list id", [self.check(id=["a.b"])], "dotted lowercase"), + ( + "a list suppression type", + [self.check(suppression={"endsWithType": []})], + "endsWithType", + ), + ( + "a hub-skipped reader before a hub file check", + [self.check(scope="baseline", hub=False), self.check(id="c.d", scope="file")], + "in the hub's run", + ), ( "ground false on a file check", [self.check(scope="baseline"), self.check(id="c.d", scope="file", ground=False)], From b66a482f944d05dfb83353bff115e9e06b435997 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 23 Oct 2026 17:29:25 -0700 Subject: [PATCH 12/20] Parse Each Workflow Once and Resolve a DriftNote Id in One Place check_interface parses its workflow once, through the same YAML helper the concurrency and dependabot reads use. driftnote_findings resolves an id with one catalog lookup and the registry, and the unused owner helper goes. The validate action's description names the audit helper suite. --- .github/actions/validate/action.yml | 2 +- spec/audit.py | 46 ++++++++++++++--------------- tests/test_audit_helpers.py | 7 +++-- 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/.github/actions/validate/action.yml b/.github/actions/validate/action.yml index 55e0de961..42adb2734 100644 --- a/.github/actions/validate/action.yml +++ b/.github/actions/validate/action.yml @@ -2,7 +2,7 @@ # The fleet doc-lint block and the Python leg in validate-task.yml, which runs the unittest suite under tests/, do not cover the rest of the hub's own machinery. # Carrying this hook makes the hub exercise the override path on every hub pull request, where a repo with no hook of its own exercises the hub default at .github/actions/validate-default. name: Validate repository (hub) -description: Registry and spec validation, the self-tests outside the unittest suite, the fleet-skills freshness check, and the unclassified-character report. +description: Registry and spec validation, the audit helper suite and the self-tests outside the unittest suite, the fleet-skills freshness check, and the unclassified-character report. runs: using: composite diff --git a/spec/audit.py b/spec/audit.py index 084412c5d..d3587c7dd 100755 --- a/spec/audit.py +++ b/spec/audit.py @@ -677,12 +677,6 @@ def catalog_owner(spec, cid): return None, any(c.get("id") == cid for c in cross) -def check_id_owner(spec, cid): - """`(owning project type or None, whether spec/project-types.json or spec/checks.json defines the id)`.""" - owner, known = catalog_owner(spec, cid) - return owner, known or bool(registered_check(cid)) - - def is_suppression_id(entry, cid): """Whether a driftNote naming `cid` suppresses a registered check's finding on this repo, rather than naming a check to judge.""" check = registered_check(cid) @@ -700,15 +694,15 @@ def driftnote_findings(entry, spec, open_count): for cid in validate.CHECK_ID_RE.findall(note): if is_suppression_id(entry, cid): continue - owner, known = check_id_owner(spec, cid) - if not known: + owner, in_catalog = catalog_owner(spec, cid) + if not (in_catalog or registered_check(cid)): out.append( ( "DRIFT", f"registry: driftNote names check '{cid}', which neither spec/checks.json nor spec/project-types.json defines - fix the id or drop the note: {quoted}", ) ) - elif not catalog_owner(spec, cid)[1]: + elif not in_catalog: out.append( ( "DRIFT", @@ -845,12 +839,19 @@ def workspace_cspell_words(text): ) -def _compose(text): - """The YAML node tree of `text`, or None where it does not parse.""" +def _yaml(text, compose=False): + """`(document, True)`, the node tree with `compose`, or `(None, False)` where `text` does not parse.""" try: - return yaml.compose(text, Loader=yaml.SafeLoader) + return ( + yaml.compose(text, Loader=yaml.SafeLoader) if compose else yaml.safe_load(text) + ), True except yaml.YAMLError: - return None + return None, False + + +def _compose(text): + """The YAML node tree of `text`, or None where it is empty or does not parse.""" + return _yaml(text, compose=True)[0] def _mapping(node): @@ -917,12 +918,11 @@ def split_jobs(text): def check_interface(path, contract, text): """DRIFT where a workflow breaks its fixed contract by name and wiring, never its body (spec/fidelity-model.md).""" - try: - yaml.compose(text, Loader=yaml.SafeLoader) - except yaml.YAMLError: + root, parsed = _yaml(text, compose=True) + if not parsed: return [("DRIFT", f"interface: {path} does not parse as YAML, so its contract is unread")] findings = [] - jobs = job_nodes(text) + jobs = _mapping(_mapping(root).get("jobs")) for k in contract.get("requiredJobKeys", []): if k not in jobs: findings.append(("DRIFT", f"interface: {path} missing required job '{k}'")) @@ -937,7 +937,7 @@ def check_interface(path, contract, text): ) ) tok = contract.get("artifactNameToken") - if tok and tok not in code_view(_compose(text)): + if tok and tok not in code_view(root): findings.append( ("DRIFT", f"interface: {path} missing the '{tok}-' artifact handoff") ) @@ -1087,9 +1087,8 @@ def concurrency_caller_findings(path, text): f"concurrency: {path} writes its concurrency in a shape the audit cannot read. Write a block mapping with group and cancel-in-progress, each a single-line scalar.", ) ] - try: - doc = yaml.safe_load(text) - except yaml.YAMLError: + doc, parsed = _yaml(text) + if not parsed: return [] if not isinstance(doc, dict): return unreadable @@ -1138,9 +1137,8 @@ def concurrency_caller_findings(path, text): def dependabot_ecosystems(text): """The package-ecosystem values a dependabot.yml declares, or None where it does not parse.""" - try: - doc = yaml.safe_load(text) - except yaml.YAMLError: + doc, parsed = _yaml(text) + if not parsed: return None updates = doc.get("updates") if isinstance(doc, dict) else None if not isinstance(updates, list): diff --git a/tests/test_audit_helpers.py b/tests/test_audit_helpers.py index 07db0b4c6..d9afd482d 100755 --- a/tests/test_audit_helpers.py +++ b/tests/test_audit_helpers.py @@ -4355,7 +4355,7 @@ def test_every_registered_check_names_a_function_of_its_scope(self) -> None: def test_every_check_function_is_registered(self) -> None: named = {audit.check_function(c).__name__ for c in audit.registered_checks()} defined = {n for n in vars(audit) if n.startswith("check_") and callable(vars(audit)[n])} - defined -= {"check_function", "check_id_owner", "check_interface", "check_verbatim"} + defined -= {"check_function", "check_interface", "check_verbatim"} defined -= {"check_intent_staleness"} self.assertEqual(defined - named, set()) @@ -4386,8 +4386,9 @@ def test_an_unparsed_dependabot_file_reads_as_unparsed(self) -> None: def test_a_registry_id_is_a_known_check_id(self) -> None: spec: dict[str, Any] = {"types": {"types": {}, "crossCutting": {}}} - self.assertEqual(audit.check_id_owner(spec, "settings.general"), (None, True)) - self.assertEqual(audit.check_id_owner(spec, "no.such-check"), (None, False)) + self.assertEqual(audit.catalog_owner(spec, "settings.general"), (None, False)) + self.assertEqual(audit.registered_check("settings.general")["id"], "settings.general") + self.assertEqual(audit.registered_check("no.such-check"), {}) if __name__ == "__main__": From 9dbf189ef96ee65169a161e60e6aed5012124f65 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 23 Oct 2026 17:48:12 -0700 Subject: [PATCH 13/20] Name What Each Tool Shares from spec/carried_text.py OPERATIONS.md says which tools share the fence step and which share the region reading, AUDIT.md's secrets runbook runs the audit through uv, and a validate.py docstring drops a pointer to code that no longer exists. --- AUDIT.md | 2 +- OPERATIONS.md | 2 +- spec/validate.py | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/AUDIT.md b/AUDIT.md index 406f763cb..603f84a31 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -101,7 +101,7 @@ Run [`WORKFLOW.md`][workflow]'s methodology against the repo's **own** Actions, - **General settings, the registry description, labels, rulesets, the Dependabot security features, the fleet project link, and deployment environments** - fetch the hub and check out `main`. Run `repo-config/configure.sh check "/" ""` from that checkout. Pass the target repository and its registry `workflowModel` explicitly. The token needs project access as well as the admin the ruleset endpoints require, since a token without it cannot read the project link and the command reports that group as failing rather than as clean, and the command's own error names the scope to grant. The host needs a runnable `py -3` or `python3` too, since a run against a hub checkout resolves the registry description through [`spec/resolve_description.py`][resolve-description] and exits before the first check when neither interpreter answers. The command checks what `configure.sh apply` writes: the declared settings, the derived settings and the registry description, the declared labels, the Dependabot security features, the shared `main` ruleset, the `develop` ruleset the model selects, and the link to the fleet project the hub declares. It reports the live `bypass_actors` list without asserting it, because bypass authority is a per-repository human decision. It also checks one group apply never writes, the deployment environments the registry's `environments` declares for the repo: each one exists, its deployment-branch policy is the form declared, and under a `custom` policy the entries it allows are exactly the declared branches, so anything added by hand reads as drift. An environment the registry declares nothing about is reported rather than asserted, since GitHub creates some on its own. Neither this command nor the **Secrets** bullet below queries an environment's own secret and variable stores, where the API exposes a secret's name but not its value and a variable's name and value both, so a clean run says nothing about whether an environment holds what a deploy needs. -- **Secrets** - from the same hub checkout, run [`spec/audit.py`][audit-runner] `""` and read the findings it prints with a `secrets:` prefix. That argument is the registry entry name rather than the `/` form `configure.sh` takes, and an `/` argument prints `Not cataloged` and exits 2. The runner reads both the Actions and the Dependabot store on every repo, and a token that cannot read either one fails that repo's whole run with an error rather than reporting names as missing. It resolves what each store must hold from the hub's own [`spec/secrets.json`][secrets] plus the registry entry's `publish[]`/`types[]`/`requiredSecrets[]`. That file's `baseline` requires its names in both stores for every fleet repo, and a mechanism adds to a store only where the mechanism's own `stores` list names that store. A declared type claims nothing where the entry's profile for it reads `lint-only`, and claims no coverage token where the tree carries no tests for it, so a repo declaring a language can still owe nothing beyond the baseline. It reports three things: a required name missing from a store, a forbidden name present in either store, and a configured name no applicable mechanism claims. A forbidden name present draws the third as well as the second, since the claimed set is built from the required names alone. Names are read, never values. A shallow hub clone fails the whole run, so no `secrets:` line prints, per section 8. +- **Secrets** - from the same hub checkout, run [`spec/audit.py`][audit-runner] as `uv run --no-project --script spec/audit.py ""` and read the findings it prints with a `secrets:` prefix. That argument is the registry entry name rather than the `/` form `configure.sh` takes, and an `/` argument prints `Not cataloged` and exits 2. The runner reads both the Actions and the Dependabot store on every repo, and a token that cannot read either one fails that repo's whole run with an error rather than reporting names as missing. It resolves what each store must hold from the hub's own [`spec/secrets.json`][secrets] plus the registry entry's `publish[]`/`types[]`/`requiredSecrets[]`. That file's `baseline` requires its names in both stores for every fleet repo, and a mechanism adds to a store only where the mechanism's own `stores` list names that store. A declared type claims nothing where the entry's profile for it reads `lint-only`, and claims no coverage token where the tree carries no tests for it, so a repo declaring a language can still owe nothing beyond the baseline. It reports three things: a required name missing from a store, a forbidden name present in either store, and a configured name no applicable mechanism claims. A forbidden name present draws the third as well as the second, since the claimed set is built from the required names alone. Names are read, never values. A shallow hub clone fails the whole run, so no `secrets:` line prints, per section 8. - **Dependabot ecosystem coverage** - for each ecosystem the repo's tree implies, `.github/dependabot.yml` must declare it: `github-actions` when `.github/workflows/` holds at least one `.yml` or `.yaml` entry (those workflows reference actions, and otherwise those versions go stale and a stood-up merge-bot has no action-update PRs to auto-merge), and `devcontainers` when a `.devcontainer` is present. The mechanical check (`spec/audit.py`) asserts each implied ecosystem's **presence**, and it runs at all only where `.github/dependabot.yml` exists and is non-empty. A missing `dependabot.yml` is a file-presence letter of its own, and an empty one is caught by neither check, since the file-presence check reports only absence, so an empty file is reported by hand. A tree-implied ecosystem that file declares nowhere is a **drift finding**. Then confirm **by inspection** that each declared ecosystem **dual-targets `main` + `develop`** per the [Branching Model][governance-branching-model], since the regex below cannot pair an ecosystem with its `target-branch`. Language ecosystems (`nuget`/`uv`/`npm`) are directory-scoped and audited by inspection too. diff --git a/OPERATIONS.md b/OPERATIONS.md index fc73cb931..ccf4d3c0d 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -139,5 +139,5 @@ The `gh api --method PATCH repos/[owner/repo]/pulls/[number]` form still works a - [catalog/](./catalog/) holds reference snippets the audit compares implementations against. - [reports/](./reports/) holds per-repo audit output. - [scripts/](./scripts/) holds the gates that run in CI and locally, and that every fleet repository reaches rather than carries. -- `extract_section()`, `strip_sections()`, `fence_step()`, and `normalize()` in [spec/carried_text.py](./spec/carried_text.py) are the shared primitives for a carried document's level-two heading regions. The audit, the carry, the skill build, and the review ledger each read through them. +- `fence_step()` in [spec/carried_text.py](./spec/carried_text.py) is the fence reading the audit, the carry, the skill build, and the review ledger share. Its `extract_section()`, `strip_sections()`, and `normalize()` are the region reading the audit and the carry share. - `read_regular_file()` and `write_regular_file()` in [host-setup/agent-safety/claude/install.py](./host-setup/agent-safety/claude/install.py) are the shared primitives for reading and replacing a file that must be regular. Each opens without blocking and judges the open descriptor, so a FIFO or a device is refused rather than hanging the call. diff --git a/spec/validate.py b/spec/validate.py index 5ce267f54..fbc584b98 100755 --- a/spec/validate.py +++ b/spec/validate.py @@ -134,8 +134,7 @@ def _bracket_matches(text, open_char, close_char): nesting, ignoring the other bracket type, needs one such map per bracket type rather than one pass mixing both. Built with a single left-to-right stack pass over the whole text rather than one depth- counting scan per open position: re-scanning from every unmatched open is what made a prior version of - this walk O(N^2) on a run of N unmatched opens, as it did in spec/audit.py's sibling - implementation. A close pops the most recently pushed open, the same pairing a fresh depth count from + this walk O(N^2) on a run of N unmatched opens. A close pops the most recently pushed open, the same pairing a fresh depth count from that open would find, so this is one pass, not N. An open with no closing partner, or a close with nothing open, is left out of the map, same as before. From 1e84fa4e3bbba0381f18214dbd010d215d51e467 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 23 Oct 2026 18:04:45 -0700 Subject: [PATCH 14/20] Hold the DriftNote Check Last and Scope the Fence-Reading Claims A test holds registry.driftnotes as the last registered check, since it counts the findings before it. OPERATIONS.md and scripts/README.md scope the shared fence reading to the audit's region extraction, and AUDIT.md states the registry-only id as the exception after its conclusion. --- AUDIT.md | 2 +- OPERATIONS.md | 2 +- scripts/README.md | 2 +- tests/test_audit_helpers.py | 4 ++++ 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/AUDIT.md b/AUDIT.md index 603f84a31..4cb9bc6b1 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -203,7 +203,7 @@ A downstream repo uses its context where it is worth most. It **files findings a **Re-running the audit needs a full hub clone with git history.** The verbatim stale-vs-modified classification and the intent staleness advisory walk the canonical's history (`git log` / `git show` from the hub root). A shallow hub clone cannot answer "matches a past hub revision". In one, `spec/audit.py` reports a single ERROR per repo and exits non-zero. `spec/audit.py --issue` exits 2 with the error on stderr. `spec/fidelity_honesty.py` stops before writing a report. Each error says to run `git fetch --unshallow origin` in that checkout. A downstream agent verifying one finding without the full history can instead compare against the current hub canonical on `main` (the whole file for a file-level unit, or the named `## heading` block for a verbatim section), which decides current-match but not stale-vs-modified. An agent picking up such an issue **re-runs the audit first and acts on the live result, not the pasted findings**: a repo moves between filing and pickup, so a stale block leads an agent to "fix" what is already fixed (re-requesting secrets that exist, attempting a no-op forward-sync). State the findings as evidence for *why* the issue was filed, never as the current state. -**Reconcile `driftNotes` in the same pass.** A registry `driftNote` records a *current* deviation from the baseline. Once the deviation is resolved the note is deleted, not left describing finished work, since hand-maintained prose drifts silently otherwise. `spec/audit.py` flags two shapes of note, neither of them gated on the rest of the audit being clean. A note asserting outstanding work in prose ("pending", "not yet", "missing", "behind", ...) is contradicted outright by a clean audit, and where findings are open it is raised as a question of which one it means, because gating the check on a clean audit meant one standing finding a repo could not clear exempted its whole note list, and the repo carrying open findings is where a stale note is most likely. A note naming the check that would retire it, as an id in parentheses and matched with them (`(hugo.generator.pinned)`, the bare id is not detected), is the mechanically checkable shape and is surfaced on **every** run: the audit resolves the id against the catalog and confirms the repo declares its type, then hands the check itself to the auditor, since section 4 above is judged by hand. An id only `spec/checks.json` defines is reported as evaluated, and the note goes once that check passes. **So a note naming a check id is retired by a person, not by a run.** Write it that way anyway. The id says exactly what would close the note, and the surfaced finding puts that decision in front of whoever runs the audit rather than leaving the note to sit until someone rereads it. A note naming a suppression id is not surfaced this way, since it suppresses a finding rather than naming a check to judge. The check it suppresses reports it once stale. A note naming `(python.directories.declared)` on a repo without the `python` type suppresses the Python discovery advisory. It is stale once no `.py` file remains. A note naming `(filenaming.scripts)` exempts each path it names as a code span from the file-naming dimension. A named path the check no longer flags is stale, and so is a note naming no path. The pending-marker check above still reads either. +**Reconcile `driftNotes` in the same pass.** A registry `driftNote` records a *current* deviation from the baseline. Once the deviation is resolved the note is deleted, not left describing finished work, since hand-maintained prose drifts silently otherwise. `spec/audit.py` flags two shapes of note, neither of them gated on the rest of the audit being clean. A note asserting outstanding work in prose ("pending", "not yet", "missing", "behind", ...) is contradicted outright by a clean audit, and where findings are open it is raised as a question of which one it means, because gating the check on a clean audit meant one standing finding a repo could not clear exempted its whole note list, and the repo carrying open findings is where a stale note is most likely. A note naming the check that would retire it, as an id in parentheses and matched with them (`(hugo.generator.pinned)`, the bare id is not detected), is the mechanically checkable shape and is surfaced on **every** run: the audit resolves the id against the catalog and confirms the repo declares its type, then hands the check itself to the auditor, since section 4 above is judged by hand. **So a note naming a check id is retired by a person, not by a run.** The one exception is an id only `spec/checks.json` defines, which is reported as evaluated, and the note goes once that check passes. Write it that way anyway. The id says exactly what would close the note, and the surfaced finding puts that decision in front of whoever runs the audit rather than leaving the note to sit until someone rereads it. A note naming a suppression id is not surfaced this way, since it suppresses a finding rather than naming a check to judge. The check it suppresses reports it once stale. A note naming `(python.directories.declared)` on a repo without the `python` type suppresses the Python discovery advisory. It is stale once no `.py` file remains. A note naming `(filenaming.scripts)` exempts each path it names as a code span from the file-naming dimension. A named path the check no longer flags is stale, and so is a note naming no path. The pending-marker check above still reads either. ## 9. Escalate diff --git a/OPERATIONS.md b/OPERATIONS.md index ccf4d3c0d..f1235b5e3 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -139,5 +139,5 @@ The `gh api --method PATCH repos/[owner/repo]/pulls/[number]` form still works a - [catalog/](./catalog/) holds reference snippets the audit compares implementations against. - [reports/](./reports/) holds per-repo audit output. - [scripts/](./scripts/) holds the gates that run in CI and locally, and that every fleet repository reaches rather than carries. -- `fence_step()` in [spec/carried_text.py](./spec/carried_text.py) is the fence reading the audit, the carry, the skill build, and the review ledger share. Its `extract_section()`, `strip_sections()`, and `normalize()` are the region reading the audit and the carry share. +- `fence_step()` in [spec/carried_text.py](./spec/carried_text.py) is the fence reading the audit's region extraction, the carry, the skill build, and the review ledger share. Its `extract_section()`, `strip_sections()`, and `normalize()` are the region reading the audit and the carry share. - `read_regular_file()` and `write_regular_file()` in [host-setup/agent-safety/claude/install.py](./host-setup/agent-safety/claude/install.py) are the shared primitives for reading and replacing a file that must be regular. Each opens without blocking and judges the open descriptor, so a FIFO or a device is refused rather than hanging the call. diff --git a/scripts/README.md b/scripts/README.md index 1a16dc1d0..3d62c6399 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -376,7 +376,7 @@ The tool validates the registry identity, target origin, feature-branch worktree The section modes close the gap between the two things that already existed: a tree mode owns a whole tree, and [`spec/audit.py`][audit] names a stale section precisely without writing anything, so the one apply step in [`RESYNC.md`][resync] with no tested tool behind it was the step the AGENTS.md-overwrite incident happened on. `check-sections` reports each declared verbatim section and which of them are stale, and exits `1` while any is. `apply-sections` replaces only those regions and then re-reads the file to assert three things: the level-two heading sequence is unchanged, every line outside the replaced regions is byte-identical to what was there before, and each declared section now matches the hub's canonical. The second of those is the one that earns its keep, since the defect it exists for, one blank line dropped between the preamble and the first heading, is invisible in a diff review, breaks no renderer, and no gate in this repository flags it. -Region boundaries come from `spec/carried_text.py`'s fence reading, which the audit shares, rather than a second reading of CommonMark, so a level-two heading marker shown inside a code sample cannot end a region for this tool and not for the check that judges the result. The replacement carries the terminator the section's own heading line carried, so a CRLF repository keeps its endings and the file is never rewritten to one ending it may not use throughout. +Region boundaries come from `spec/carried_text.py`'s fence reading, which the audit's region extraction shares, rather than a second reading of CommonMark, so a level-two heading marker shown inside a code sample cannot end a region for this tool and not for the check that judges the result. The replacement carries the terminator the section's own heading line carried, so a CRLF repository keeps its endings and the file is never rewritten to one ending it may not use throughout. Every refusal is raised while planning, before the first file is written, so a refused run leaves nothing re-vendored. What refuses: a section the manifest declares that the repository does not carry (where it belongs is a standup question), a file absent from the target, a file that is not UTF-8 or that carries a bare carriage return, which the fidelity comparison reads as a line break and this tool does not, a file declaring both placeholders and verbatim sections (a verbatim unit carries none, per [`spec/fidelity-model.md`][fidelity-model]), a region holding content the fidelity comparison normalizes away, such as a Dependabot-owned action pin, whose bytes the repository rather than the hub owns, and two declared names resolving to one region. A failure after that point is the write itself or its assertion, and it names the file it was working on, because an earlier file may already hold its re-vendored content. diff --git a/tests/test_audit_helpers.py b/tests/test_audit_helpers.py index d9afd482d..a3ffd8e73 100755 --- a/tests/test_audit_helpers.py +++ b/tests/test_audit_helpers.py @@ -4368,6 +4368,10 @@ def test_suppression_ids_resolve_against_the_registry(self) -> None: self.assertTrue(audit.is_suppression_id({"types": ["python"]}, "filenaming.scripts")) self.assertFalse(audit.is_suppression_id(typed, "settings.general")) + def test_the_driftnote_check_runs_last(self) -> None: + # It counts the findings before it, so a check registered after it would go uncounted. + self.assertEqual(audit.registered_checks()[-1]["id"], "registry.driftnotes") + def test_the_smoke_selftest_passes(self) -> None: with contextlib.redirect_stdout(io.StringIO()) as out: self.assertEqual(audit._selftest_cases(), 0, out.getvalue()) From 24e6b7fc301dd258cbeede327db3892fd7c9009b Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 23 Oct 2026 18:24:02 -0700 Subject: [PATCH 15/20] Point at description_findings Where It Now Lives GOVERNANCE.md, spec/validate.py, and spec/resolve_description.py name spec/audit_markdown.py as description_findings' home. The concurrency refusal names the shapes the parser actually refuses, and a test helper module drops a standard-library-only claim that no longer holds. --- GOVERNANCE.md | 2 +- spec/audit.py | 2 +- spec/resolve_description.py | 2 +- spec/validate.py | 4 ++-- tests/workflow_steps.py | 3 +-- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/GOVERNANCE.md b/GOVERNANCE.md index a524c8c35..93558713d 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -356,7 +356,7 @@ Contributors commit to this repo with signed commits. The SSH-signing setup live Every repo's GitHub repository details (the About panel) follow a fixed convention so the fleet stays consistent and self-describing. -- **Description** is one canonical sentence that carries to the README, the About panel, and (for a Docker repo) the Docker Hub short description alike, at most **100 characters**, Docker Hub's short-description cap and the tightest surface it feeds. Once a repo declares `registry/repos.json`'s optional `description` field, that field is the source, itself link-free plain text on one line for the same reason: `repo-config/configure.sh apply` writes it to the About panel directly, and the README's **tagline** (its first non-empty line after the `#` H1 heading) follows it rather than the other way around. A repo that has not adopted the field yet keeps the pre-existing convention, where the README tagline is the source of truth and the About panel is set from it by hand (`gh api -X PATCH repos// -f description=...`). `spec/audit.py`'s `description_findings()` reports drift either way, falling back to the tagline when no field is declared. It is that one line and not the paragraph it opens: a README may carry further paragraphs below the tagline, and no mirror reads them. When the current description is *more specific* than the declared source (a chip revision or variant it omits), surface the drift to the maintainer rather than silently discarding the detail, and the fix is to sharpen the declared source so the other mirrors follow it. Docker Hub receives it from the About panel, which the docker-readme task reads at publish time, so an About panel left diverged from the canonical value is carried onward rather than corrected there. +- **Description** is one canonical sentence that carries to the README, the About panel, and (for a Docker repo) the Docker Hub short description alike, at most **100 characters**, Docker Hub's short-description cap and the tightest surface it feeds. Once a repo declares `registry/repos.json`'s optional `description` field, that field is the source, itself link-free plain text on one line for the same reason: `repo-config/configure.sh apply` writes it to the About panel directly, and the README's **tagline** (its first non-empty line after the `#` H1 heading) follows it rather than the other way around. A repo that has not adopted the field yet keeps the pre-existing convention, where the README tagline is the source of truth and the About panel is set from it by hand (`gh api -X PATCH repos// -f description=...`). `spec/audit_markdown.py`'s `description_findings()`, which the audit runs, reports drift either way, falling back to the tagline when no field is declared. It is that one line and not the paragraph it opens: a README may carry further paragraphs below the tagline, and no mirror reads them. When the current description is *more specific* than the declared source (a chip revision or variant it omits), surface the drift to the maintainer rather than silently discarding the detail, and the fix is to sharpen the declared source so the other mirrors follow it. Docker Hub receives it from the About panel, which the docker-readme task reads at publish time, so an About panel left diverged from the canonical value is carried onward rather than corrected there. - **Topics** are optional, and any that are present match the repo's actual content. Do not invent topics to fill the field. - **Include in the home page**: Releases on, Deployments off, Packages off. These toggles are UI-only, since the REST and GraphQL APIs neither read nor write them, so they are set by hand and cannot be audited through `gh`. diff --git a/spec/audit.py b/spec/audit.py index d3587c7dd..4b9227d8d 100755 --- a/spec/audit.py +++ b/spec/audit.py @@ -1084,7 +1084,7 @@ def concurrency_caller_findings(path, text): unreadable = [ ( "DRIFT", - f"concurrency: {path} writes its concurrency in a shape the audit cannot read. Write a block mapping with group and cancel-in-progress, each a single-line scalar.", + f"concurrency: {path} writes its concurrency in a shape the audit cannot read. Give it a mapping whose group and cancel-in-progress are scalars, or a group scalar alone.", ) ] doc, parsed = _yaml(text) diff --git a/spec/resolve_description.py b/spec/resolve_description.py index 55e62584f..df69f07e3 100755 --- a/spec/resolve_description.py +++ b/spec/resolve_description.py @@ -2,7 +2,7 @@ """Resolve one repo's declared registry/repos.json description, for repo-config/configure.sh. Delegates to description_errors() (validate.py) so configure.sh validates a declared description -against the exact same contract spec/audit.py's description_findings() does, rather than a third +against the exact same contract spec/audit_markdown.py's description_findings() does, rather than a third hand-rolled copy of the same rules. Prints the declared description to stdout and exits 0 when the repo has no declared description diff --git a/spec/validate.py b/spec/validate.py index fbc584b98..5b4bc297e 100755 --- a/spec/validate.py +++ b/spec/validate.py @@ -534,7 +534,7 @@ def description_errors(name, desc): Absence is not checked here, since the field is optional. `desc` is only passed in once a repo declares it. - Link-free is enforced because spec/audit.py's description_findings() strips Markdown links from the README's + Link-free is enforced because spec/audit_markdown.py's description_findings() strips Markdown links from the README's own tagline before comparing, but never re-strips the declared field it compares that tagline against. A declared value carrying a link would therefore report as a permanent readme mismatch, and repo-config/ configure.sh would push the literal Markdown source to GitHub's About panel, which does not render it. @@ -1150,7 +1150,7 @@ def check_secret_set(label, entry, need_kind): ) # Optional, since a repo whose callers pass no python-directories input declares nothing. errors.extend(python_directories_errors_for_repo(repo, name)) - # Optional per GOVERNANCE.md "Repository Details": a repo that has not adopted the field yet is unaffected, since spec/audit.py's description_findings() falls back to the README tagline for it. + # Optional per GOVERNANCE.md "Repository Details": a repo that has not adopted the field yet is unaffected, since spec/audit_markdown.py's description_findings() falls back to the README tagline for it. errors.extend(description_errors_for_repo(repo, name)) # Optional, and read by repo-config/configure.sh's check mode rather than by any check here. # CI runs no JSON-schema validation, so the schema's own required-fields and branches-iff-custom rules are enforced here or nowhere. diff --git a/tests/workflow_steps.py b/tests/workflow_steps.py index 5a72bfa27..ba29d4bfa 100644 --- a/tests/workflow_steps.py +++ b/tests/workflow_steps.py @@ -1,7 +1,6 @@ """Read one named step out of a workflow or composite action, bounded to that step. -The suites run on the standard library alone, so the readers work by indentation. A step is -sliced out before any key is looked for in it, so a step missing a key reads as missing rather +The readers work by indentation. A step is sliced out before any key is looked for in it, so a step missing a key reads as missing rather than as the next step's key. """ From 2c4cc9c5d72b76c4c660abcd79a50729498a046c Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 23 Oct 2026 18:39:03 -0700 Subject: [PATCH 16/20] State What a Registry-Only DriftNote Id and CI's Audit Suites Do AUDIT.md says a registry-only id's finding names the check the audit evaluates and the note goes once that check reports nothing, rather than calling it an exception the run decides. OPERATIONS.md names both audit suites CI runs. --- AUDIT.md | 2 +- OPERATIONS.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/AUDIT.md b/AUDIT.md index 4cb9bc6b1..5afb6a580 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -203,7 +203,7 @@ A downstream repo uses its context where it is worth most. It **files findings a **Re-running the audit needs a full hub clone with git history.** The verbatim stale-vs-modified classification and the intent staleness advisory walk the canonical's history (`git log` / `git show` from the hub root). A shallow hub clone cannot answer "matches a past hub revision". In one, `spec/audit.py` reports a single ERROR per repo and exits non-zero. `spec/audit.py --issue` exits 2 with the error on stderr. `spec/fidelity_honesty.py` stops before writing a report. Each error says to run `git fetch --unshallow origin` in that checkout. A downstream agent verifying one finding without the full history can instead compare against the current hub canonical on `main` (the whole file for a file-level unit, or the named `## heading` block for a verbatim section), which decides current-match but not stale-vs-modified. An agent picking up such an issue **re-runs the audit first and acts on the live result, not the pasted findings**: a repo moves between filing and pickup, so a stale block leads an agent to "fix" what is already fixed (re-requesting secrets that exist, attempting a no-op forward-sync). State the findings as evidence for *why* the issue was filed, never as the current state. -**Reconcile `driftNotes` in the same pass.** A registry `driftNote` records a *current* deviation from the baseline. Once the deviation is resolved the note is deleted, not left describing finished work, since hand-maintained prose drifts silently otherwise. `spec/audit.py` flags two shapes of note, neither of them gated on the rest of the audit being clean. A note asserting outstanding work in prose ("pending", "not yet", "missing", "behind", ...) is contradicted outright by a clean audit, and where findings are open it is raised as a question of which one it means, because gating the check on a clean audit meant one standing finding a repo could not clear exempted its whole note list, and the repo carrying open findings is where a stale note is most likely. A note naming the check that would retire it, as an id in parentheses and matched with them (`(hugo.generator.pinned)`, the bare id is not detected), is the mechanically checkable shape and is surfaced on **every** run: the audit resolves the id against the catalog and confirms the repo declares its type, then hands the check itself to the auditor, since section 4 above is judged by hand. **So a note naming a check id is retired by a person, not by a run.** The one exception is an id only `spec/checks.json` defines, which is reported as evaluated, and the note goes once that check passes. Write it that way anyway. The id says exactly what would close the note, and the surfaced finding puts that decision in front of whoever runs the audit rather than leaving the note to sit until someone rereads it. A note naming a suppression id is not surfaced this way, since it suppresses a finding rather than naming a check to judge. The check it suppresses reports it once stale. A note naming `(python.directories.declared)` on a repo without the `python` type suppresses the Python discovery advisory. It is stale once no `.py` file remains. A note naming `(filenaming.scripts)` exempts each path it names as a code span from the file-naming dimension. A named path the check no longer flags is stale, and so is a note naming no path. The pending-marker check above still reads either. +**Reconcile `driftNotes` in the same pass.** A registry `driftNote` records a *current* deviation from the baseline. Once the deviation is resolved the note is deleted, not left describing finished work, since hand-maintained prose drifts silently otherwise. `spec/audit.py` flags two shapes of note, neither of them gated on the rest of the audit being clean. A note asserting outstanding work in prose ("pending", "not yet", "missing", "behind", ...) is contradicted outright by a clean audit, and where findings are open it is raised as a question of which one it means, because gating the check on a clean audit meant one standing finding a repo could not clear exempted its whole note list, and the repo carrying open findings is where a stale note is most likely. A note naming the check that would retire it, as an id in parentheses and matched with them (`(hugo.generator.pinned)`, the bare id is not detected), is the mechanically checkable shape and is surfaced on **every** run: the audit resolves the id against the catalog and confirms the repo declares its type, then hands the check itself to the auditor, since section 4 above is judged by hand. **So a note naming a check id is retired by a person, not by a run.** For an id only `spec/checks.json` defines, the finding says the audit evaluates it, and the note goes once that check reports nothing. Write it that way anyway. The id says exactly what would close the note, and the surfaced finding puts that decision in front of whoever runs the audit rather than leaving the note to sit until someone rereads it. A note naming a suppression id is not surfaced this way, since it suppresses a finding rather than naming a check to judge. The check it suppresses reports it once stale. A note naming `(python.directories.declared)` on a repo without the `python` type suppresses the Python discovery advisory. It is stale once no `.py` file remains. A note naming `(filenaming.scripts)` exempts each path it names as a code span from the file-naming dimension. A named path the check no longer flags is stale, and so is a note naming no path. The pending-marker check above still reads either. ## 9. Escalate diff --git a/OPERATIONS.md b/OPERATIONS.md index f1235b5e3..24c9e1a08 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -4,7 +4,7 @@ How this repository is run. It ships no application code, so its operations are ## Local Verification -What verifying a change here requires, including the part CI cannot perform. The gates below do run in CI, so a local run of them buys an earlier failure rather than a different one, with two exceptions stated where each belongs, the prose gate in the next section and `test_install.py` below the block. The two verifications CI never performs are the fleet audit and the repository-configuration check, because each reads live state in another repository over the API and a pull request runner is given neither the credentials nor a target to read. `tests/test_audit_helpers.py` is what CI runs of the audit, which exercises the checker against its fixtures and reads no repository at all. So a change to [spec/](./spec/), [registry/](./registry/), [repo-config/](./repo-config/) or [AUDIT.md](./AUDIT.md) is verified by running `uv run --no-project --script spec/audit.py` and `repo-config/configure.sh check` from this checkout before the pull request opens, each from the repository root and each taking the arguments the two runbooks below give it. A green pipeline says nothing about either, and taking it as coverage is the failure this section exists to name. +What verifying a change here requires, including the part CI cannot perform. The gates below do run in CI, so a local run of them buys an earlier failure rather than a different one, with two exceptions stated where each belongs, the prose gate in the next section and `test_install.py` below the block. The two verifications CI never performs are the fleet audit and the repository-configuration check, because each reads live state in another repository over the API and a pull request runner is given neither the credentials nor a target to read. `tests/test_audit_helpers.py` and `tests/test_audit.py` are what CI runs of the audit. They exercise the checker against fixtures and a faked gh, and read no repository at all. So a change to [spec/](./spec/), [registry/](./registry/), [repo-config/](./repo-config/) or [AUDIT.md](./AUDIT.md) is verified by running `uv run --no-project --script spec/audit.py` and `repo-config/configure.sh check` from this checkout before the pull request opens, each from the repository root and each taking the arguments the two runbooks below give it. A green pipeline says nothing about either, and taking it as coverage is the failure this section exists to name. ### Run the gates locally, which is wider than CI From df6eb4efad218a655b87542647325c14aa56a57c Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 23 Oct 2026 18:52:55 -0700 Subject: [PATCH 17/20] Test a Declared Dependabot Ecosystem and Narrow the Carry Claim A positive case holds dependabot_ecosystems reading each declared package-ecosystem, and OPERATIONS.md says carry.py shares only the comparable form and the fence step from spec/carried_text.py. --- OPERATIONS.md | 2 +- tests/test_audit_helpers.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/OPERATIONS.md b/OPERATIONS.md index 24c9e1a08..5e8a38078 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -139,5 +139,5 @@ The `gh api --method PATCH repos/[owner/repo]/pulls/[number]` form still works a - [catalog/](./catalog/) holds reference snippets the audit compares implementations against. - [reports/](./reports/) holds per-repo audit output. - [scripts/](./scripts/) holds the gates that run in CI and locally, and that every fleet repository reaches rather than carries. -- `fence_step()` in [spec/carried_text.py](./spec/carried_text.py) is the fence reading the audit's region extraction, the carry, the skill build, and the review ledger share. Its `extract_section()`, `strip_sections()`, and `normalize()` are the region reading the audit and the carry share. +- `fence_step()` in [spec/carried_text.py](./spec/carried_text.py) is the fence reading the audit's region extraction, the carry, the skill build, and the review ledger share. Its `extract_section()` and `strip_sections()` are the audit's region reading, and its `normalize()` is the comparable form the audit and the carry share. - `read_regular_file()` and `write_regular_file()` in [host-setup/agent-safety/claude/install.py](./host-setup/agent-safety/claude/install.py) are the shared primitives for reading and replacing a file that must be regular. Each opens without blocking and judges the open descriptor, so a FIFO or a device is refused rather than hanging the call. diff --git a/tests/test_audit_helpers.py b/tests/test_audit_helpers.py index a3ffd8e73..2efd32717 100755 --- a/tests/test_audit_helpers.py +++ b/tests/test_audit_helpers.py @@ -4387,6 +4387,8 @@ def test_a_registry_only_id_reads_as_evaluated_and_a_catalog_id_does_not(self) - def test_an_unparsed_dependabot_file_reads_as_unparsed(self) -> None: self.assertIsNone(audit.dependabot_ecosystems("updates:\n - [\n")) self.assertEqual(audit.dependabot_ecosystems("updates: []\n"), set()) + declared = "updates:\n - package-ecosystem: github-actions\n - package-ecosystem: 'uv'\n" + self.assertEqual(audit.dependabot_ecosystems(declared), {"github-actions", "uv"}) def test_a_registry_id_is_a_known_check_id(self) -> None: spec: dict[str, Any] = {"types": {"types": {}, "crossCutting": {}}} From 9487e8e20cbea33feac3c263d8098b73e87d2f84 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 23 Oct 2026 19:13:39 -0700 Subject: [PATCH 18/20] Keep a Tagline's Code Spans as Written and Pin the Registry Wiring strip_md_links takes a code span from the source rather than rebuilding it, since the parser trims its padding. New tests pin which checks skip the hub, which baseline slot each file check reads, and split_jobs at a sibling key and at a file with no final newline. build_dist.py states the reason it imports spec/carried_text.py lazily as it now stands. --- scripts/build_dist.py | 4 ++-- spec/audit_markdown.py | 8 ++++++-- tests/test_audit_helpers.py | 34 +++++++++++++++++++++++++++++++++- tests/test_build_dist.py | 5 ++--- 4 files changed, 43 insertions(+), 8 deletions(-) diff --git a/scripts/build_dist.py b/scripts/build_dist.py index db058b9af..95365c58e 100755 --- a/scripts/build_dist.py +++ b/scripts/build_dist.py @@ -194,8 +194,8 @@ def write_plugin_manifest(names): def _carried_text(): """spec/carried_text.py, imported on first use rather than at module import, then held. - Imported lazily because scripts/skills_install.py imports this module on hosts whose Python - predates what the spec scripts need, and installing never fills a region. Held because the caller + Imported lazily because scripts/skills_install.py imports this module and installing never + fills a region, so the install path loads nothing from spec/. Held because the caller below reaches for it once per line, and re-resolving it there costs a sys.path scan and a sys.modules lookup every time: one pass over the 24 authored skills, 4644 lines across the 40 Markdown files they comprise, measured warm in one process, best of five, costs 17.5 ms diff --git a/spec/audit_markdown.py b/spec/audit_markdown.py index 656518ecf..949e3219f 100644 --- a/spec/audit_markdown.py +++ b/spec/audit_markdown.py @@ -72,12 +72,16 @@ def strip_md_links(text, references=None): tokens = _MD.parseInline(text, {"references": dict(references or {})})[0].children or [] if not any(tok.type in ("link_open", "image") for tok in tokens): return text - out = [] + out, pos = [], 0 for tok in tokens: if tok.type in ("text", "html_inline", "image"): out.append(tok.content) elif tok.type == "code_inline": - out.append(tok.markup + tok.content + tok.markup) + # The parser trims a code span's padding, so the span is taken from the source as written. + fence = re.escape(tok.markup) + span = re.compile(f"{fence} ?{re.escape(tok.content)} ?{fence}").search(text, pos) + out.append(span.group(0) if span else tok.markup + tok.content + tok.markup) + pos = span.end() if span else pos elif tok.type == "softbreak": out.append("\n") elif tok.type not in ("link_open", "link_close"): diff --git a/tests/test_audit_helpers.py b/tests/test_audit_helpers.py index 2efd32717..a4c8d6403 100755 --- a/tests/test_audit_helpers.py +++ b/tests/test_audit_helpers.py @@ -599,6 +599,17 @@ def test_job_level_environment_as_the_first_key(self) -> None: class SplitJobsCase(unittest.TestCase): """Holds the split_jobs edge cases.""" + def test_a_sibling_key_ends_the_last_job(self) -> None: + text = "jobs:\n a:\n runs-on: x\n\n b:\n runs-on: y\nenv:\n K: v\n" + self.assertEqual( + audit.split_jobs(text), {"a": " a:\n runs-on: x\n\n", "b": " b:\n runs-on: y\n"} + ) + + def test_the_last_job_keeps_a_final_line_with_no_newline(self) -> None: + self.assertEqual( + audit.split_jobs("jobs:\n a:\n runs-on: x"), {"a": " a:\n runs-on: x"} + ) + def test_trailing_comment(self) -> None: trailing = audit.split_jobs(REL_OK + "# a trailing top-level comment\n") self.assertEqual(set(trailing), {"get-version", "build-widget", "github-release"}) @@ -1383,8 +1394,10 @@ def test_strip_md_links(self) -> None: self.assertEqual( audit_markdown.strip_md_links("Plain intro line."), "Plain intro line.", msg ) - padded = "Runs ` make ` targets with ``a `b` c``." + padded = "Runs ` make ` targets with `` `b` ``." self.assertEqual(audit_markdown.strip_md_links(padded), padded, msg) + linked_span = padded + " See [docs](u)." + self.assertEqual(audit_markdown.strip_md_links(linked_span), padded + " See docs.", msg) imaged = "Cleans media ![ci](https://img.example/b.svg) fast." self.assertEqual(audit_markdown.strip_md_links(imaged), "Cleans media ci fast.", msg) self.assertEqual(audit_markdown.strip_md_links(nested), "See API [docs] for details.", msg) @@ -4368,6 +4381,25 @@ def test_suppression_ids_resolve_against_the_registry(self) -> None: self.assertTrue(audit.is_suppression_id({"types": ["python"]}, "filenaming.scripts")) self.assertFalse(audit.is_suppression_id(typed, "settings.general")) + def test_only_these_checks_skip_the_hub(self) -> None: + skipped = {c["id"] for c in audit.registered_checks() if not c["hub"]} + want = {"sections.undeclared", "carried.template-ref", "tree.verbatim", "hubonly.carried"} + self.assertEqual(skipped, want) + + def test_file_checks_read_their_own_baseline_slots(self) -> None: + run = audit.Run({"name": "Widget"}, {}, "o/r", "main", {}, None, None) + run.__dict__["hub"] = False + run.__dict__["baseline"] = {"AGENTS.md": (None, {"Absent"}, {"Pinned"})} + run.content["AGENTS.md"] = "## Pinned\n\nacme-hub\n" + presence = audit.check_sections_presence(run, "AGENTS.md") + self.assertEqual(len(presence), 1) + self.assertIn("'Absent'", presence[0][1]) + with mock.patch.object(audit, "check_verbatim", lambda label, *_, **__: [("DRIFT", label)]): + verbatim = audit.check_sections_verbatim(run, "AGENTS.md") + self.assertEqual(verbatim, [("DRIFT", "AGENTS.md section 'Pinned'")]) + with mock.patch.object(audit, "hub_repo_name", lambda: "acme-hub"): + self.assertEqual(audit.check_carried_template_ref(run, "AGENTS.md"), []) + def test_the_driftnote_check_runs_last(self) -> None: # It counts the findings before it, so a check registered after it would go uncounted. self.assertEqual(audit.registered_checks()[-1]["id"], "registry.driftnotes") diff --git a/tests/test_build_dist.py b/tests/test_build_dist.py index a3d60427a..cb1da5a51 100755 --- a/tests/test_build_dist.py +++ b/tests/test_build_dist.py @@ -946,9 +946,8 @@ def test_carried_text_is_resolved_once_however_many_lines_are_read(self) -> None def test_importing_this_module_does_not_import_carried_text(self) -> None: """The laziness `scripts/skills_install.py` depends on, which caching must not give up. - It imports this module on hosts whose Python predates what the spec scripts need, and installing - never fills a region, so importing `carried_text` at module scope would break that path. Run in a - fresh interpreter because this test process has already imported it by other means. + It imports this module and installing never fills a region, so the install path loads nothing + from spec/. Run in a fresh interpreter because this test process has already imported it by other means. The measurement is the delta across the import rather than the child's absolute state, since a `sitecustomize` or `usercustomize` hook runs before `-c` does and could import a module of From d32d3065129a2b7219bf5212eb49cdf5d6a06cfd Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 23 Oct 2026 19:29:58 -0700 Subject: [PATCH 19/20] Test the Hub's Own Run, an Unparsed Dependabot File, and on: Shapes audit_repo run as the hub skips only the hub-false checks and still runs the file checks after them. check_dependabot_ecosystems reports an unparseable file as one finding, and the schedule gate reads a list and a scalar on: as well as a block mapping. --- tests/test_audit_helpers.py | 62 +++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tests/test_audit_helpers.py b/tests/test_audit_helpers.py index a4c8d6403..3aaa99042 100755 --- a/tests/test_audit_helpers.py +++ b/tests/test_audit_helpers.py @@ -3,6 +3,7 @@ from __future__ import annotations +import base64 import contextlib import inspect import io @@ -4400,6 +4401,67 @@ def test_file_checks_read_their_own_baseline_slots(self) -> None: with mock.patch.object(audit, "hub_repo_name", lambda: "acme-hub"): self.assertEqual(audit.check_carried_template_ref(run, "AGENTS.md"), []) + def test_the_hub_run_skips_only_the_hub_false_checks(self) -> None: + seen: list[str] = [] + + def recorder(check: dict[str, Any]) -> Callable[..., list[Any]]: + def run_check(run: Any, *path: str) -> list[Any]: + seen.append(check["id"]) + if check["scope"] == "baseline": + run.content[path[0]] = "" + return [] + + return run_check + + head = {"commit": {"sha": "c0ffee", "commit": {"tree": {"sha": "t"}}}} + + def gh(path: str, ok404: bool = False) -> Any: + return {"private": False} if path == "repos/acme/Hub" else head + + entry = {"name": "Hub", "url": "https://github.com/acme/Hub", "types": ["docs"]} + spec = {"registry": {"defaults": {}}, "files": {"baseline": [{"path": "AGENTS.md"}]}} + with ( + mock.patch.object(audit, "gh", gh), + mock.patch.object(audit, "hub_repo_name", lambda: "Hub"), + mock.patch.object(audit, "check_function", recorder), + ): + audit.audit_repo(entry, spec) + skipped = { + "sections.undeclared", + "carried.template-ref", + "tree.verbatim", + "hubonly.carried", + } + self.assertEqual(skipped & set(seen), set()) + self.assertIn("carried.version-literal", seen) + self.assertEqual(seen[-1], "registry.driftnotes") + + def test_an_unparsed_dependabot_file_is_one_finding(self) -> None: + run = audit.Run({"name": "Widget"}, {}, "o/r", "main", {}, None, None) + bad = {"encoding": "base64", "content": base64.b64encode(b"updates:\n - [\n").decode()} + + def gh(path: str, ok404: bool = False) -> Any: + return bad if path.endswith("dependabot.yml?ref=main") else None + + with mock.patch.object(audit, "gh", gh): + got = audit.check_dependabot_ecosystems(run) + want = "dependabot: .github/dependabot.yml does not parse as YAML, so its ecosystems are unread" + self.assertEqual(got, [("DRIFT", want)]) + + def test_a_schedule_trigger_reads_in_each_on_shape(self) -> None: + job = ( + "jobs:\n publish:\n uses: acme/hub/.github/workflows/build-release-task.yml@x\n" + " if: ${{ github.event_name == 'workflow_dispatch' }}\n" + ) + for on, want in ( + ("on: [schedule, workflow_dispatch]\n", True), + ("on: schedule\n", True), + ("on: [schedule, push]\n", False), + ): + with self.subTest(on): + doc = audit.yaml.safe_load(on + job) + self.assertEqual(audit._schedule_never_publishes(doc), want) + def test_the_driftnote_check_runs_last(self) -> None: # It counts the findings before it, so a check registered after it would go uncounted. self.assertEqual(audit.registered_checks()[-1]["id"], "registry.driftnotes") From b914f083bf696743319c8175f0c84016ad594af0 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 23 Oct 2026 20:00:29 -0700 Subject: [PATCH 20/20] Hold strip_md_links Linear by Ratio Rather Than Wall Clock The CI unit job runs under coverage, where markdown-it's pass over 20000 unmatched brackets took 1.08 s against the 1 s bound. The test now times 5000 and 20000 brackets and holds the larger under eight times the smaller, which a quadratic reader exceeds and a linear one does not. --- tests/test_audit_helpers.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/test_audit_helpers.py b/tests/test_audit_helpers.py index 3aaa99042..a95fbc221 100755 --- a/tests/test_audit_helpers.py +++ b/tests/test_audit_helpers.py @@ -1408,14 +1408,17 @@ def test_strip_md_links(self) -> None: ) def test_strip_md_links_stays_linear(self) -> None: - pathological = "[" * 20000 - start = time.monotonic() - audit_markdown.strip_md_links(pathological) - elapsed = time.monotonic() - start - self.assertLessEqual( - elapsed, - 1.0, - f"description: strip_md_links took {elapsed:.2f}s on unmatched brackets, expected linear", + # A ratio rather than a wall-clock bound, since coverage tracing slows both runs alike. + def cost(n: int) -> float: + start = time.monotonic() + audit_markdown.strip_md_links("[" * n) + return time.monotonic() - start + + small, large = cost(5000), cost(20000) + self.assertLess( + large, + 8 * small + 0.05, + f"description: strip_md_links took {small:.2f}s then {large:.2f}s on 4x the brackets, expected linear", ) def test_workspace_cspell_words(self) -> None: