diff --git a/.github/ISSUE_TEMPLATE/release_checklist.yml b/.github/ISSUE_TEMPLATE/release_checklist.yml index d3d1ed86e..27aab78b5 100644 --- a/.github/ISSUE_TEMPLATE/release_checklist.yml +++ b/.github/ISSUE_TEMPLATE/release_checklist.yml @@ -18,9 +18,18 @@ body: attributes: label: GitHub / nuget.org (one-time or per repo) options: - - label: Trusted Publishing registered for `release.yml` (and `release-nuget.yml` if NuGet-only OIDC) — see `docs/PUBLISHING.md` + - label: Trusted Publishing registered on nuget.org for workflow file `reusable-release-nuget.yml` — the REUSABLE workflow that runs `NuGet/login`, not `release.yml` or `release-nuget.yml`; a policy on a caller does not match (`docs/PUBLISHING.md`). `NUGET_USER` is the nuget.org profile name of whoever created the policy - label: Repo variables set (`NUGET_PUBLISH_MODE`, optional staging/post-push/SBOM) — `docs/GitHubRepoVariables.md` - label: Secrets present (`NUGET_USER` for OIDC, `NUGET_API_KEY` if apikey mode, optional `NUGET_STAGING_API_KEY`, `RELEASE_NOTIFICATION_WEBHOOK_URL`) + - type: checkboxes + id: before_tag + attributes: + label: Before you tag (docs/RELEASE_RUNBOOK.md steps 4-9) + options: + - label: "Step 4: VERSION, ci/published-version and consumer-template's AshlarConsumerPackageVersion bumped together on the release commit" + - label: "Steps 5-7: consumer trust rows rewritten, published-version docs bumped, PublicAPI Unshipped promoted to Shipped" + - label: "Step 8: rehearsed with a prerelease (release-nuget.yml, X.Y.Z-rc1) and the whole post-push chain went green" + - label: "Step 9: bash scripts/release/readiness-verdict-for-sha.sh printed VERIFIED and exited 0 for the exact commit being tagged" - type: checkboxes id: ship attributes: diff --git a/.github/workflows/compose-gate.yml b/.github/workflows/compose-gate.yml index 6c36142b4..6a377e3ac 100644 --- a/.github/workflows/compose-gate.yml +++ b/.github/workflows/compose-gate.yml @@ -39,6 +39,10 @@ jobs: set -euo pipefail mkdir -p test-results docker compose -f deploy/compose/docker-compose.test.yml up --build --abort-on-container-exit --exit-code-from test-ubuntu test-ubuntu + # The service's pinned filter (FullyQualifiedName~BaseFrameworkSmokeTests) exits 0 when it + # matches nothing, and the service then writes ok:true whether or not a test ran. Its TRX + # lands here through the ../../test-results volume; the guard fails the step if it ran nothing. + bash scripts/ci/zero-test-guard.sh test-results/ubuntu-baseframework.trx test -f test-results/ubuntu-results.json test -f test-results/ubuntu-logs.txt @@ -77,6 +81,7 @@ jobs: path: | test-results/ubuntu-results.json test-results/ubuntu-logs.txt + test-results/ubuntu-baseframework.trx ${{ runner.temp }}/compose-test-summary.txt retention-days: 14 diff --git a/.github/workflows/cross-platform-tests.yml b/.github/workflows/cross-platform-tests.yml index 1828ad7de..8f6571c56 100644 --- a/.github/workflows/cross-platform-tests.yml +++ b/.github/workflows/cross-platform-tests.yml @@ -55,6 +55,10 @@ on: # - 'src/Ashlar.Tests.Application/**' # - 'application/src/Ashlar.Tests.CLI/**' # +# Every filtered `dotnet test` below pins -f and writes its own TRX, and each step ends with +# scripts/ci/zero-test-guard.sh over those files. Before, each ran both target frameworks under ONE +# LogFileName, so net10.0's TRX overwrote net8.0's ("WARNING: Overwriting results file") and a +# framework whose filter matched nothing left no trace in the file the publish step read. jobs: cross-platform: name: Build & Test - ${{ matrix.os }} @@ -86,117 +90,224 @@ jobs: shell: bash run: | dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj \ + -f net8.0 \ --no-build \ --blame-hang-timeout 450s \ --filter "(FullyQualifiedName~BaseFrameworkSmokeTests|FullyQualifiedName~RuntimeStudioBlackBoxSmokeTests)" \ --logger "console;verbosity=minimal" \ - --logger "trx;LogFileName=smoke-${{ matrix.os }}.trx" \ + --logger "trx;LogFileName=smoke-net8.0-${{ matrix.os }}.trx" \ + --results-directory test-results + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj \ + -f net10.0 \ + --no-build \ + --blame-hang-timeout 450s \ + --filter "(FullyQualifiedName~BaseFrameworkSmokeTests|FullyQualifiedName~RuntimeStudioBlackBoxSmokeTests)" \ + --logger "console;verbosity=minimal" \ + --logger "trx;LogFileName=smoke-net10.0-${{ matrix.os }}.trx" \ --results-directory test-results dotnet test src/Ashlar.Tests.BackgroundAgents/Ashlar.Tests.BackgroundAgents.csproj \ + -f net8.0 \ --no-build \ --blame-hang-timeout 60s \ --filter "FullyQualifiedName~RuntimeStudioWhiteBoxSmokeTests" \ --logger "console;verbosity=minimal" \ - --logger "trx;LogFileName=smoke-whitebox-${{ matrix.os }}.trx" \ + --logger "trx;LogFileName=smoke-whitebox-net8.0-${{ matrix.os }}.trx" \ --results-directory test-results + dotnet test src/Ashlar.Tests.BackgroundAgents/Ashlar.Tests.BackgroundAgents.csproj \ + -f net10.0 \ + --no-build \ + --blame-hang-timeout 60s \ + --filter "FullyQualifiedName~RuntimeStudioWhiteBoxSmokeTests" \ + --logger "console;verbosity=minimal" \ + --logger "trx;LogFileName=smoke-whitebox-net10.0-${{ matrix.os }}.trx" \ + --results-directory test-results + bash scripts/ci/zero-test-guard.sh test-results/smoke-net8.0-${{ matrix.os }}.trx test-results/smoke-net10.0-${{ matrix.os }}.trx test-results/smoke-whitebox-net8.0-${{ matrix.os }}.trx test-results/smoke-whitebox-net10.0-${{ matrix.os }}.trx - name: Run tests (playground) if: github.event.inputs.scope == 'playground' shell: bash run: | dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj \ + -f net8.0 \ --no-build \ --blame-hang-timeout 900s \ --filter "Category=Playground" \ --logger "console;verbosity=minimal" \ - --logger "trx;LogFileName=playground-infra-${{ matrix.os }}.trx" \ + --logger "trx;LogFileName=playground-infra-net8.0-${{ matrix.os }}.trx" \ + --results-directory test-results + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj \ + -f net10.0 \ + --no-build \ + --blame-hang-timeout 900s \ + --filter "Category=Playground" \ + --logger "console;verbosity=minimal" \ + --logger "trx;LogFileName=playground-infra-net10.0-${{ matrix.os }}.trx" \ + --results-directory test-results + dotnet test src/Ashlar.Tests.BackgroundAgents/Ashlar.Tests.BackgroundAgents.csproj \ + -f net8.0 \ + --no-build \ + --blame-hang-timeout 300s \ + --filter "Category=Playground" \ + --logger "console;verbosity=minimal" \ + --logger "trx;LogFileName=playground-bg-net8.0-${{ matrix.os }}.trx" \ --results-directory test-results dotnet test src/Ashlar.Tests.BackgroundAgents/Ashlar.Tests.BackgroundAgents.csproj \ + -f net10.0 \ --no-build \ --blame-hang-timeout 300s \ --filter "Category=Playground" \ --logger "console;verbosity=minimal" \ - --logger "trx;LogFileName=playground-bg-${{ matrix.os }}.trx" \ + --logger "trx;LogFileName=playground-bg-net10.0-${{ matrix.os }}.trx" \ --results-directory test-results + bash scripts/ci/zero-test-guard.sh test-results/playground-infra-net8.0-${{ matrix.os }}.trx test-results/playground-infra-net10.0-${{ matrix.os }}.trx test-results/playground-bg-net8.0-${{ matrix.os }}.trx test-results/playground-bg-net10.0-${{ matrix.os }}.trx - name: Run tests (dogfood) if: github.event.inputs.scope == 'dogfood' shell: bash run: | dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj \ + -f net8.0 \ + --no-build \ + --blame-hang-timeout 180s \ + --filter "FullyQualifiedName~Dogfood" \ + --logger "console;verbosity=minimal" \ + --logger "trx;LogFileName=dogfood-net8.0-${{ matrix.os }}.trx" \ + --results-directory test-results + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj \ + -f net10.0 \ --no-build \ --blame-hang-timeout 180s \ --filter "FullyQualifiedName~Dogfood" \ --logger "console;verbosity=minimal" \ - --logger "trx;LogFileName=dogfood-${{ matrix.os }}.trx" \ + --logger "trx;LogFileName=dogfood-net10.0-${{ matrix.os }}.trx" \ --results-directory test-results + bash scripts/ci/zero-test-guard.sh test-results/dogfood-net8.0-${{ matrix.os }}.trx test-results/dogfood-net10.0-${{ matrix.os }}.trx - name: Run tests (integration) if: github.event.inputs.scope == 'integration' || github.event.inputs.scope == 'full' shell: bash run: | dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj \ + -f net8.0 \ + --no-build \ + --blame-hang-timeout 720s \ + --filter "Category=Integration" \ + --logger "console;verbosity=minimal" \ + --logger "trx;LogFileName=integration-net8.0-${{ matrix.os }}.trx" \ + --results-directory test-results + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj \ + -f net10.0 \ --no-build \ --blame-hang-timeout 720s \ --filter "Category=Integration" \ --logger "console;verbosity=minimal" \ - --logger "trx;LogFileName=integration-${{ matrix.os }}.trx" \ + --logger "trx;LogFileName=integration-net10.0-${{ matrix.os }}.trx" \ --results-directory test-results + bash scripts/ci/zero-test-guard.sh test-results/integration-net8.0-${{ matrix.os }}.trx test-results/integration-net10.0-${{ matrix.os }}.trx - name: Run tests (persistence) if: github.event.inputs.scope == 'persistence' || github.event.inputs.scope == 'full' shell: bash run: | dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj \ + -f net8.0 \ + --no-build \ + --blame-hang-timeout 60s \ + --filter "FullyQualifiedName~InMemoryPersistenceTests" \ + --logger "console;verbosity=minimal" \ + --logger "trx;LogFileName=persistence-net8.0-${{ matrix.os }}.trx" \ + --results-directory test-results + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj \ + -f net10.0 \ --no-build \ --blame-hang-timeout 60s \ --filter "FullyQualifiedName~InMemoryPersistenceTests" \ --logger "console;verbosity=minimal" \ - --logger "trx;LogFileName=persistence-${{ matrix.os }}.trx" \ + --logger "trx;LogFileName=persistence-net10.0-${{ matrix.os }}.trx" \ --results-directory test-results + bash scripts/ci/zero-test-guard.sh test-results/persistence-net8.0-${{ matrix.os }}.trx test-results/persistence-net10.0-${{ matrix.os }}.trx - name: Run tests (e2e) if: github.event.inputs.scope == 'e2e' || github.event.inputs.scope == 'full' shell: bash run: | dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj \ + -f net8.0 \ --no-build \ --blame-hang-timeout 900s \ --filter "Category=E2E" \ --logger "console;verbosity=minimal" \ - --logger "trx;LogFileName=e2e-${{ matrix.os }}.trx" \ + --logger "trx;LogFileName=e2e-net8.0-${{ matrix.os }}.trx" \ --results-directory test-results + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj \ + -f net10.0 \ + --no-build \ + --blame-hang-timeout 900s \ + --filter "Category=E2E" \ + --logger "console;verbosity=minimal" \ + --logger "trx;LogFileName=e2e-net10.0-${{ matrix.os }}.trx" \ + --results-directory test-results + bash scripts/ci/zero-test-guard.sh test-results/e2e-net8.0-${{ matrix.os }}.trx test-results/e2e-net10.0-${{ matrix.os }}.trx - name: Run tests (adaptation) if: github.event.inputs.scope == 'adaptation' shell: bash run: | dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj \ + -f net8.0 \ --no-build \ --blame-hang-timeout 90s \ --filter "Category=Adaptation" \ --logger "console;verbosity=minimal" \ - --logger "trx;LogFileName=adaptation-${{ matrix.os }}.trx" \ + --logger "trx;LogFileName=adaptation-net8.0-${{ matrix.os }}.trx" \ --results-directory test-results + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj \ + -f net10.0 \ + --no-build \ + --blame-hang-timeout 90s \ + --filter "Category=Adaptation" \ + --logger "console;verbosity=minimal" \ + --logger "trx;LogFileName=adaptation-net10.0-${{ matrix.os }}.trx" \ + --results-directory test-results + bash scripts/ci/zero-test-guard.sh test-results/adaptation-net8.0-${{ matrix.os }}.trx test-results/adaptation-net10.0-${{ matrix.os }}.trx - name: Run tests (trust) if: github.event.inputs.scope == 'trust' || github.event.inputs.scope == 'full' shell: bash run: | dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj \ + -f net8.0 \ + --no-build \ + --blame-hang-timeout 180s \ + --filter "FullyQualifiedName~Trust" \ + --logger "console;verbosity=minimal" \ + --logger "trx;LogFileName=trust-infra-net8.0-${{ matrix.os }}.trx" \ + --results-directory test-results + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj \ + -f net10.0 \ --no-build \ --blame-hang-timeout 180s \ --filter "FullyQualifiedName~Trust" \ --logger "console;verbosity=minimal" \ - --logger "trx;LogFileName=trust-infra-${{ matrix.os }}.trx" \ + --logger "trx;LogFileName=trust-infra-net10.0-${{ matrix.os }}.trx" \ + --results-directory test-results + dotnet test src/Ashlar.Tests.BackgroundAgents/Ashlar.Tests.BackgroundAgents.csproj \ + -f net8.0 \ + --no-build \ + --blame-hang-timeout 60s \ + --filter "FullyQualifiedName~Trust" \ + --logger "console;verbosity=minimal" \ + --logger "trx;LogFileName=trust-bg-net8.0-${{ matrix.os }}.trx" \ --results-directory test-results dotnet test src/Ashlar.Tests.BackgroundAgents/Ashlar.Tests.BackgroundAgents.csproj \ + -f net10.0 \ --no-build \ --blame-hang-timeout 60s \ --filter "FullyQualifiedName~Trust" \ --logger "console;verbosity=minimal" \ - --logger "trx;LogFileName=trust-bg-${{ matrix.os }}.trx" \ + --logger "trx;LogFileName=trust-bg-net10.0-${{ matrix.os }}.trx" \ --results-directory test-results + bash scripts/ci/zero-test-guard.sh test-results/trust-infra-net8.0-${{ matrix.os }}.trx test-results/trust-infra-net10.0-${{ matrix.os }}.trx test-results/trust-bg-net8.0-${{ matrix.os }}.trx test-results/trust-bg-net10.0-${{ matrix.os }}.trx - name: Run tests (full Infrastructure) if: github.event.inputs.scope == 'full' diff --git a/.github/workflows/distribution-matrix-gate.yml b/.github/workflows/distribution-matrix-gate.yml index 91b6974ff..ec55f1b2f 100644 --- a/.github/workflows/distribution-matrix-gate.yml +++ b/.github/workflows/distribution-matrix-gate.yml @@ -191,7 +191,9 @@ jobs: dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj \ -f net10.0 \ --filter "FullyQualifiedName~Ashlar.Tests.Infrastructure.Tests.VirtualProduction.FrameworkVirtualProdDemosTests.Virtual_prod_IAshlarClient_GetStatusAsync_matches_console_and_blazor_demos" \ + --logger "trx;LogFileName=virtual-prod-client.trx" --results-directory test-results \ --verbosity minimal + bash scripts/ci/zero-test-guard.sh test-results/virtual-prod-client.trx pack-hosting-graph-alignment: name: Pack script vs Ashlar.Hosting graph diff --git a/.github/workflows/full-platform-readiness-gate.yml b/.github/workflows/full-platform-readiness-gate.yml index c1eb7d9a6..28671a1c0 100644 --- a/.github/workflows/full-platform-readiness-gate.yml +++ b/.github/workflows/full-platform-readiness-gate.yml @@ -73,6 +73,48 @@ on: - "scripts/ci/test-commercial-receipts.py" - "src/Ashlar.Tests.Orchestration/**" - "src/Ashlar.Tests.Kernel/**" + # The rest of the ProjectReference closure of every test project `ashlar validate` discovers + # (the native lanes reach it through `ashlar ci verify`). Each directory below is built into + # at least one of those suites, so before this block editing it skipped the lanes that run + # them. ReadinessClosureConventionTests (cert-gate) recomputes the closure from the tree and + # fails naming any project directory or build input that either list does not cover. + - "application/src/Ashlar.API/**" + - "products/ashlar-cloud/src/Ashlar.Cloud/**" + - "products/ashlar-cluster/src/Ashlar.Cluster/**" + - "products/ashlar-native/src/Ashlar.Native/**" + - "products/ashlar-workstation/src/Ashlar.Workstation/**" + - "products/tests/Ashlar.Tests.Products/**" + - "samples/hello-brick/HelloBrick/**" + - "samples/hello-brick/HelloBrick.Tests/**" + - "src/Ashlar.Adapters.Models/**" + - "src/Ashlar.Agents.TestKit/**" + - "src/Ashlar.Analyzers.Tests/**" + - "src/Ashlar.Authoring/**" + - "src/Ashlar.BackgroundAgents.HostRunners/**" + - "src/Ashlar.Bricks.Owasp/**" + - "src/Ashlar.Bricks.SqlProfile/**" + - "src/Ashlar.Certification.State/**" + - "src/Ashlar.Client/**" + - "src/Ashlar.Contracts/**" + - "src/Ashlar.Framework.Sdk/**" + - "src/Ashlar.Ingress.AwsSns/**" + - "src/Ashlar.Ingress.AwsSns.Tests/**" + - "src/Ashlar.Ingress.DynamoDb/**" + - "src/Ashlar.Ingress.DynamoDb.Tests/**" + - "src/Ashlar.Mcp.Client/**" + - "src/Ashlar.Mcp.Client.Tests/**" + - "src/Ashlar.Mcp.Server/**" + - "src/Ashlar.Mcp.Server.Tests/**" + - "src/Ashlar.Sdk/**" + - "src/Ashlar.Tests.Application/**" + - "src/Ashlar.Tests.Contracts/**" + - "src/Ashlar.Tests.Domain/**" + - "src/Ashlar.Tests.Transport/**" + - "src/Ashlar.Transport.A2A/**" + - "src/Ashlar.Transport.A2A.Server/**" + - "src/Ashlar.Transport.A2A.Server.Tests/**" + - "src/Ashlar.Transport.A2A.Tests/**" + - "src/Ashlar.Transport.Grpc.Server/**" - "scripts/pack-ashlar-hosting-graph.sh" - "scripts/pack-ashlar-hosting-graph.ps1" - "scripts/verify-stable-sdk-host-sample-packages.sh" @@ -85,6 +127,17 @@ on: - "VERSION" - "Directory.Build.targets" - "Directory.Packages.props" + # Build inputs of that closure that are not project directories: nested Directory.Build.props + # that MSBuild imports implicitly, the root .editorconfig the compiler reads, and the files + # closure projects compile, embed or copy from outside their own directory (src/Ashlar.Compat + # is compiled into most of them through src/Directory.Build.props and Directory.Build.targets). + - ".editorconfig" + - "application/Directory.Build.props" + - "products/Directory.Build.props" + - "samples/Directory.Build.props" + - "src/Ashlar.Compat/**" + - "samples/templates/brick/**" + - "samples/physical-atom-cert/**" # Merge queue. A required check that does not run on `merge_group` never reports, and the # queue waits on it forever - so this is deliberately unfiltered, exactly like the # `pull_request` trigger above. @@ -178,6 +231,45 @@ jobs: "scripts/ci/test-commercial-receipts.py" "src/Ashlar.Tests.Orchestration/**" "src/Ashlar.Tests.Kernel/**" + # The rest of the ProjectReference closure of the suites `ashlar validate` discovers - see + # the matching note in the push trigger above. + "application/src/Ashlar.API/**" + "products/ashlar-cloud/src/Ashlar.Cloud/**" + "products/ashlar-cluster/src/Ashlar.Cluster/**" + "products/ashlar-native/src/Ashlar.Native/**" + "products/ashlar-workstation/src/Ashlar.Workstation/**" + "products/tests/Ashlar.Tests.Products/**" + "samples/hello-brick/HelloBrick/**" + "samples/hello-brick/HelloBrick.Tests/**" + "src/Ashlar.Adapters.Models/**" + "src/Ashlar.Agents.TestKit/**" + "src/Ashlar.Analyzers.Tests/**" + "src/Ashlar.Authoring/**" + "src/Ashlar.BackgroundAgents.HostRunners/**" + "src/Ashlar.Bricks.Owasp/**" + "src/Ashlar.Bricks.SqlProfile/**" + "src/Ashlar.Certification.State/**" + "src/Ashlar.Client/**" + "src/Ashlar.Contracts/**" + "src/Ashlar.Framework.Sdk/**" + "src/Ashlar.Ingress.AwsSns/**" + "src/Ashlar.Ingress.AwsSns.Tests/**" + "src/Ashlar.Ingress.DynamoDb/**" + "src/Ashlar.Ingress.DynamoDb.Tests/**" + "src/Ashlar.Mcp.Client/**" + "src/Ashlar.Mcp.Client.Tests/**" + "src/Ashlar.Mcp.Server/**" + "src/Ashlar.Mcp.Server.Tests/**" + "src/Ashlar.Sdk/**" + "src/Ashlar.Tests.Application/**" + "src/Ashlar.Tests.Contracts/**" + "src/Ashlar.Tests.Domain/**" + "src/Ashlar.Tests.Transport/**" + "src/Ashlar.Transport.A2A/**" + "src/Ashlar.Transport.A2A.Server/**" + "src/Ashlar.Transport.A2A.Server.Tests/**" + "src/Ashlar.Transport.A2A.Tests/**" + "src/Ashlar.Transport.Grpc.Server/**" "scripts/pack-ashlar-hosting-graph.sh" "scripts/pack-ashlar-hosting-graph.ps1" "scripts/verify-stable-sdk-host-sample-packages.sh" @@ -190,6 +282,14 @@ jobs: "VERSION" "Directory.Build.targets" "Directory.Packages.props" + # Implicit and cross-directory build inputs of that closure - see the push trigger above. + ".editorconfig" + "application/Directory.Build.props" + "products/Directory.Build.props" + "samples/Directory.Build.props" + "src/Ashlar.Compat/**" + "samples/templates/brick/**" + "samples/physical-atom-cert/**" ) if [[ -z "${BASE_SHA:-}" || -z "${HEAD_SHA:-}" ]]; then @@ -614,6 +714,10 @@ jobs: # required context and failing on a skip would deadlock any PR that touches no core path - so the # green tick alone cannot tell them apart. Anything that wants to CLAIM platform verification for # a SHA (a release sign-off, a runbook step) must read this, not the check conclusion. + # Job outputs are visible only to later jobs in this run, and the step summary is not exposed + # through the Checks API (output.summary is null on this check run). The "Readiness verdict" + # annotation emitted below IS readable from outside the run; that is what + # scripts/release/readiness-verdict-for-sha.sh reads. outputs: verdict: ${{ steps.summary.outputs.verdict }} lanes_ran: ${{ steps.summary.outputs.lanes_ran }} @@ -704,4 +808,15 @@ jobs: echo "lanes_skipped=${LANES_SKIPPED}" } >> "$GITHUB_OUTPUT" + # The one copy of the verdict anything outside this run can read through the API: an + # annotation on this check run (GET /repos/{owner}/{repo}/check-runs/{id}/annotations). + # The reader matches the title and parses the message field by field, so change neither + # without scripts/release/readiness-verdict-for-sha.py (its MESSAGE pattern) and + # tests/scripts/readiness-verdict.test.sh, which round-trips this exact line through the + # release sign-off. Emitted before the exit, like the outputs above, so a failing run + # publishes "failed" instead of nothing. At most five notices come from this step (four + # skipped lanes plus this one), well inside GitHub's annotation caps; an annotation over a + # cap is dropped silently, and the reader would then refuse the run as missing-annotation. + echo "::notice title=Readiness verdict::verdict=${VERDICT} lanes_ran=${LANES_RAN} lanes_skipped=${LANES_SKIPPED}" + exit $OVERALL diff --git a/.github/workflows/grpc-transport-gate.yml b/.github/workflows/grpc-transport-gate.yml index 37051d58f..82f22ba01 100644 --- a/.github/workflows/grpc-transport-gate.yml +++ b/.github/workflows/grpc-transport-gate.yml @@ -35,4 +35,6 @@ jobs: set -euo pipefail dotnet test src/Ashlar.Tests.Transport/Ashlar.Tests.Transport.csproj \ --filter "Category=ProdStyle" \ + --logger "trx;LogFileName=transport-prodstyle.trx" --results-directory test-results \ --blame-hang-timeout 120s + bash scripts/ci/zero-test-guard.sh test-results/transport-prodstyle.trx diff --git a/.github/workflows/mcp-a2a-gate.yml b/.github/workflows/mcp-a2a-gate.yml index 1d225683b..7abf0db22 100644 --- a/.github/workflows/mcp-a2a-gate.yml +++ b/.github/workflows/mcp-a2a-gate.yml @@ -107,4 +107,6 @@ jobs: dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj \ -f net10.0 \ --filter "FullyQualifiedName~McpA2AProtocolIngress|FullyQualifiedName~AirGappedProfileApiHostProdStyleTests" \ + --logger "trx;LogFileName=protocol-prodstyle.trx" --results-directory test-results \ --blame-hang-timeout 720s + bash scripts/ci/zero-test-guard.sh test-results/protocol-prodstyle.trx diff --git a/.github/workflows/perf-certification.yml b/.github/workflows/perf-certification.yml index 643dfb4da..80a41bdbc 100644 --- a/.github/workflows/perf-certification.yml +++ b/.github/workflows/perf-certification.yml @@ -33,19 +33,43 @@ jobs: set -euo pipefail mkdir -p perf-results echo "{\"baseline\":\"${{ github.event.inputs.baseline }}\",\"runId\":\"${{ github.run_id }}\",\"sha\":\"${{ github.sha }}\",\"timestamp\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}" > perf-results/perf-certification-meta.json + # One invocation over the whole solution, so every test project runs (most match nothing). + # It used ONE LogFileName in ONE results directory, and each project's TRX overwrote the + # previous one ("WARNING: Overwriting results file"): the file that survived was whichever + # project finished last, and the metrics step below read that. LogFilePrefix names each TRX + # __.trx, and leaving --results-directory unset keeps each + # project's under its own TestResults/, so two projects finishing in the same second cannot + # collide either (measured: two projects in one run both named theirs + # perf-certification_net8.0_20260930163958.trx). They are then gathered under perf-results/trx, + # one file per project and framework, and guarded in --solution mode: at least one project + # must have executed a test. + # + # The gather runs BEFORE the test's exit code is honoured. A failing run is the one whose + # results matter most, and under `set -e` a bare `dotnet test` would exit before the copy: + # the metrics step (if: always()) would then find no TRX and report 0 tests, 0 failed, and + # the uploaded artifact would carry none. `|| rc=$?` keeps the code; it is re-raised below. + rc=0 dotnet test Ashlar.sln \ --configuration Release \ - --logger "trx;LogFileName=perf-certification.trx" \ - --results-directory perf-results \ + --logger "trx;LogFilePrefix=perf-certification" \ --filter "FullyQualifiedName~Ashlar.Tests.Orchestration.Performance|FullyQualifiedName~Benchmark" \ - --verbosity minimal + --verbosity minimal || rc=$? + mkdir -p perf-results/trx + while IFS= read -r -d '' trx; do + project="$(basename "$(dirname "$(dirname "$trx")")")" + cp "$trx" "perf-results/trx/${project}__$(basename "$trx")" + done < <(find . -path '*/TestResults/perf-certification_*.trx' -not -path './perf-results/*' -print0) + if [[ "$rc" -ne 0 ]]; then + echo "::error::dotnet test exited ${rc}; its TRX files are kept in perf-results/trx" + exit "$rc" + fi + bash scripts/ci/zero-test-guard.sh --solution perf-results/trx/*__perf-certification_*.trx - name: Extract perf metrics and check thresholds if: always() shell: bash run: | set -euo pipefail - TRX_FILE=$(find perf-results -name '*.trx' -print -quit 2>/dev/null || true) REPORT_FILE="perf-results/perf-certification-report.json" TOTAL=0 @@ -53,11 +77,16 @@ jobs: FAILED=0 DURATION_MS=0 - if [[ -n "$TRX_FILE" && -f "$TRX_FILE" ]]; then - TOTAL=$(grep -c '/dev/null || echo 0) - PASSED=$(grep -c 'outcome="Passed"' "$TRX_FILE" 2>/dev/null || echo 0) - FAILED=$(grep -c 'outcome="Failed"' "$TRX_FILE" 2>/dev/null || echo 0) - fi + # Summed over every project's TRX. This read ONE file (find -print -quit), and with every + # project writing the same name that file was whichever project finished last. + # `grep -c` prints 0 AND exits 1 on no match, so `|| echo 0` produced "00"; `|| true` + # keeps the single 0. + for TRX_FILE in perf-results/trx/*.trx; do + [[ -f "$TRX_FILE" ]] || continue + TOTAL=$(( TOTAL + $(grep -c '` reads it and exits 0 only for a VERIFIED read from GitHub; the release runbook requires it before tagging (step 9). +- **The readiness trigger covers the whole build closure of the suites it runs.** The routing gap left open in 0.2.0 (see the 0.2.0 notes) is closed: 41 directories added across #684 and this change, and `ReadinessClosureConventionTests` (cert-gate) fails naming any closure directory either path list misses. Files read through MSBuild property functions are not traced. + +### Fixed + +- Records that named the wrong runner or gate: `ci/test-ownership.tsv`, the SPEC-006 conformance paragraph, the May readiness sign-off sheets (each PASS now annotated with its evidence), two docs that placed `MeshTaskExecutionServiceTests` wrongly, and the release checklist, which pointed nuget.org Trusted Publishing at `release.yml` instead of `reusable-release-nuget.yml`. ## [0.2.0] - 2026-09-29 diff --git a/Makefile b/Makefile index 5e1ef8ead..95cb4b65c 100644 --- a/Makefile +++ b/Makefile @@ -68,9 +68,12 @@ prod-dry-run-agent-server: # one `ashlar ci verify` runs, and it sat at 120s over 480s nets until that check existed. test-prod-style: dotnet build src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -v minimal + rm -f test-results/test-prod-style/test-prod-style.trx ASHLAR_ALLOW_MOCK=1 dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net8.0 --no-build \ --filter "Category=ProdStyle&FullyQualifiedName!~ForgeEndpointsTests&FullyQualifiedName!~FrameworkVirtualProdDemosTests" \ + --logger "trx;LogFileName=test-prod-style.trx" --results-directory test-results/test-prod-style \ --blame-hang-timeout 720s --blame-hang-dump-type none + bash scripts/ci/zero-test-guard.sh test-results/test-prod-style/test-prod-style.trx # Runs test-prod-style then the full LocalDevCore test slice (Domain + Infrastructure + CLI harness). # ProdStyle runs once in test-prod-style; the second pass excludes Category=ProdStyle. @@ -194,12 +197,17 @@ meai-pipeline-gate: kernel-gate: dotnet build Ashlar.Runtime.sln -v minimal dotnet build src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -v minimal + rm -f test-results/kernel-gate/kernel-gate-hosting.trx test-results/kernel-gate/kernel-gate-pipeline.trx ASHLAR_ALLOW_MOCK=1 dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net8.0 --no-build \ --filter "FullyQualifiedName~KernelPhaseResolutionTests|FullyQualifiedName~HostingDeploymentProfileTests|FullyQualifiedName~HostingE2ESmokeTests" \ + --logger "trx;LogFileName=kernel-gate-hosting.trx" --results-directory test-results/kernel-gate \ --blame-hang-timeout 180s --blame-hang-dump-type none + bash scripts/ci/zero-test-guard.sh test-results/kernel-gate/kernel-gate-hosting.trx ASHLAR_ALLOW_MOCK=1 dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net8.0 --no-build \ --filter "FullyQualifiedName~PipelineTemplateValidatorTests|FullyQualifiedName~PipelineLifecycleE2ETests" \ + --logger "trx;LogFileName=kernel-gate-pipeline.trx" --results-directory test-results/kernel-gate \ --blame-hang-timeout 180s --blame-hang-dump-type none + bash scripts/ci/zero-test-guard.sh test-results/kernel-gate/kernel-gate-pipeline.trx $(MAKE) meai-pipeline-gate @if [ "$${KERNEL_GATE_PRODSTYLE:-0}" = "1" ]; then $(MAKE) test-prod-style; fi @if [ "$${KERNEL_GATE_MESH:-0}" = "1" ]; then $(MAKE) mesh-lab-verify; fi @@ -433,60 +441,96 @@ ashlar-ready-gate: validate-safe: @bash scripts/validate-safe.sh +# Every dogfood target runs each target framework as its OWN invocation with its own TRX, then +# scripts/ci/zero-test-guard.sh over both. One invocation without -f runs both frameworks and +# the second overwrites the first's TRX, so a framework that matched nothing would be invisible. # Dogfood Block 1: verify observation pipeline watches Ashlar's own dev workflow dogfood-block1: dotnet build src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -v minimal - dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj --filter "FullyQualifiedName~DogfoodBlock1Tests" --no-build -v minimal + rm -f test-results/dogfood/dogfood-block1-net8.0.trx test-results/dogfood/dogfood-block1-net10.0.trx + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net8.0 --filter "FullyQualifiedName~DogfoodBlock1Tests" --no-build -v minimal --logger "trx;LogFileName=dogfood-block1-net8.0.trx" --results-directory test-results/dogfood + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net10.0 --filter "FullyQualifiedName~DogfoodBlock1Tests" --no-build -v minimal --logger "trx;LogFileName=dogfood-block1-net10.0.trx" --results-directory test-results/dogfood + bash scripts/ci/zero-test-guard.sh test-results/dogfood/dogfood-block1-net8.0.trx test-results/dogfood/dogfood-block1-net10.0.trx # Dogfood Block 2: verify static analyzer runs against Block 1 (Observation) code dogfood-block2: dotnet build src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -v minimal - dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj --filter "FullyQualifiedName~DogfoodBlock2Tests" --no-build -v minimal + rm -f test-results/dogfood/dogfood-block2-net8.0.trx test-results/dogfood/dogfood-block2-net10.0.trx + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net8.0 --filter "FullyQualifiedName~DogfoodBlock2Tests" --no-build -v minimal --logger "trx;LogFileName=dogfood-block2-net8.0.trx" --results-directory test-results/dogfood + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net10.0 --filter "FullyQualifiedName~DogfoodBlock2Tests" --no-build -v minimal --logger "trx;LogFileName=dogfood-block2-net10.0.trx" --results-directory test-results/dogfood + bash scripts/ci/zero-test-guard.sh test-results/dogfood/dogfood-block2-net8.0.trx test-results/dogfood/dogfood-block2-net10.0.trx # Dogfood Block 3: adaptation engine decomposes/recompiles Ashlar brick dogfood-block3: dotnet build src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -v minimal - dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj --filter "FullyQualifiedName~DogfoodBlock3Tests" --no-build -v minimal + rm -f test-results/dogfood/dogfood-block3-net8.0.trx test-results/dogfood/dogfood-block3-net10.0.trx + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net8.0 --filter "FullyQualifiedName~DogfoodBlock3Tests" --no-build -v minimal --logger "trx;LogFileName=dogfood-block3-net8.0.trx" --results-directory test-results/dogfood + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net10.0 --filter "FullyQualifiedName~DogfoodBlock3Tests" --no-build -v minimal --logger "trx;LogFileName=dogfood-block3-net10.0.trx" --results-directory test-results/dogfood + bash scripts/ci/zero-test-guard.sh test-results/dogfood/dogfood-block3-net8.0.trx test-results/dogfood/dogfood-block3-net10.0.trx # Dogfood Block 4: promote Ashlar fix via inheritance dogfood-block4: dotnet build src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -v minimal - dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj --filter "FullyQualifiedName~DogfoodBlock4Tests" --no-build -v minimal + rm -f test-results/dogfood/dogfood-block4-net8.0.trx test-results/dogfood/dogfood-block4-net10.0.trx + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net8.0 --filter "FullyQualifiedName~DogfoodBlock4Tests" --no-build -v minimal --logger "trx;LogFileName=dogfood-block4-net8.0.trx" --results-directory test-results/dogfood + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net10.0 --filter "FullyQualifiedName~DogfoodBlock4Tests" --no-build -v minimal --logger "trx;LogFileName=dogfood-block4-net10.0.trx" --results-directory test-results/dogfood + bash scripts/ci/zero-test-guard.sh test-results/dogfood/dogfood-block4-net8.0.trx test-results/dogfood/dogfood-block4-net10.0.trx # Dogfood Block 5: autonomy controls on Ashlar dev workflow dogfood-block5: dotnet build src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -v minimal - dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj --filter "FullyQualifiedName~DogfoodBlock5Tests" --no-build -v minimal + rm -f test-results/dogfood/dogfood-block5-net8.0.trx test-results/dogfood/dogfood-block5-net10.0.trx + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net8.0 --filter "FullyQualifiedName~DogfoodBlock5Tests" --no-build -v minimal --logger "trx;LogFileName=dogfood-block5-net8.0.trx" --results-directory test-results/dogfood + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net10.0 --filter "FullyQualifiedName~DogfoodBlock5Tests" --no-build -v minimal --logger "trx;LogFileName=dogfood-block5-net10.0.trx" --results-directory test-results/dogfood + bash scripts/ci/zero-test-guard.sh test-results/dogfood/dogfood-block5-net8.0.trx test-results/dogfood/dogfood-block5-net10.0.trx # Dogfood Block 6: SelfContextAssembler answers 24h question dogfood-block6: dotnet build src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -v minimal - dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj --filter "FullyQualifiedName~DogfoodBlock6Tests" --no-build -v minimal + rm -f test-results/dogfood/dogfood-block6-net8.0.trx test-results/dogfood/dogfood-block6-net10.0.trx + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net8.0 --filter "FullyQualifiedName~DogfoodBlock6Tests" --no-build -v minimal --logger "trx;LogFileName=dogfood-block6-net8.0.trx" --results-directory test-results/dogfood + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net10.0 --filter "FullyQualifiedName~DogfoodBlock6Tests" --no-build -v minimal --logger "trx;LogFileName=dogfood-block6-net10.0.trx" --results-directory test-results/dogfood + bash scripts/ci/zero-test-guard.sh test-results/dogfood/dogfood-block6-net8.0.trx test-results/dogfood/dogfood-block6-net10.0.trx # Dogfood Block 7: Composition engine composes for Ashlar problem dogfood-block7: dotnet build src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -v minimal - dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj --filter "FullyQualifiedName~DogfoodBlock7Tests" --no-build -v minimal + rm -f test-results/dogfood/dogfood-block7-net8.0.trx test-results/dogfood/dogfood-block7-net10.0.trx + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net8.0 --filter "FullyQualifiedName~DogfoodBlock7Tests" --no-build -v minimal --logger "trx;LogFileName=dogfood-block7-net8.0.trx" --results-directory test-results/dogfood + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net10.0 --filter "FullyQualifiedName~DogfoodBlock7Tests" --no-build -v minimal --logger "trx;LogFileName=dogfood-block7-net10.0.trx" --results-directory test-results/dogfood + bash scripts/ci/zero-test-guard.sh test-results/dogfood/dogfood-block7-net8.0.trx test-results/dogfood/dogfood-block7-net10.0.trx # Dogfood Block 8: Parallel test matrix against Ashlar tests dogfood-block8: dotnet build src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -v minimal - dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj --filter "FullyQualifiedName~DogfoodBlock8Tests" --no-build -v minimal + rm -f test-results/dogfood/dogfood-block8-net8.0.trx test-results/dogfood/dogfood-block8-net10.0.trx + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net8.0 --filter "FullyQualifiedName~DogfoodBlock8Tests" --no-build -v minimal --logger "trx;LogFileName=dogfood-block8-net8.0.trx" --results-directory test-results/dogfood + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net10.0 --filter "FullyQualifiedName~DogfoodBlock8Tests" --no-build -v minimal --logger "trx;LogFileName=dogfood-block8-net10.0.trx" --results-directory test-results/dogfood + bash scripts/ci/zero-test-guard.sh test-results/dogfood/dogfood-block8-net8.0.trx test-results/dogfood/dogfood-block8-net10.0.trx # Phase D: Composition-driven testing (Block 7–8) dogfood-block8-composed: dotnet build src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -v minimal - dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj --filter "FullyQualifiedName~DogfoodBlock8ComposedTests" --no-build -v minimal + rm -f test-results/dogfood/dogfood-block8-composed-net8.0.trx test-results/dogfood/dogfood-block8-composed-net10.0.trx + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net8.0 --filter "FullyQualifiedName~DogfoodBlock8ComposedTests" --no-build -v minimal --logger "trx;LogFileName=dogfood-block8-composed-net8.0.trx" --results-directory test-results/dogfood + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net10.0 --filter "FullyQualifiedName~DogfoodBlock8ComposedTests" --no-build -v minimal --logger "trx;LogFileName=dogfood-block8-composed-net10.0.trx" --results-directory test-results/dogfood + bash scripts/ci/zero-test-guard.sh test-results/dogfood/dogfood-block8-composed-net8.0.trx test-results/dogfood/dogfood-block8-composed-net10.0.trx # Dogfood Block 9: Instance mesh discover/advertise dogfood-block9: dotnet build src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -v minimal - dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj --filter "FullyQualifiedName~DogfoodBlock9Tests" --no-build -v minimal + rm -f test-results/dogfood/dogfood-block9-net8.0.trx test-results/dogfood/dogfood-block9-net10.0.trx + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net8.0 --filter "FullyQualifiedName~DogfoodBlock9Tests" --no-build -v minimal --logger "trx;LogFileName=dogfood-block9-net8.0.trx" --results-directory test-results/dogfood + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net10.0 --filter "FullyQualifiedName~DogfoodBlock9Tests" --no-build -v minimal --logger "trx;LogFileName=dogfood-block9-net10.0.trx" --results-directory test-results/dogfood + bash scripts/ci/zero-test-guard.sh test-results/dogfood/dogfood-block9-net8.0.trx test-results/dogfood/dogfood-block9-net10.0.trx # Phase E: Local IPC mesh - two instances share capability dogfood-block9-ipc: dotnet build src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -v minimal - dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj --filter "FullyQualifiedName~DogfoodBlock9LocalIpcTests" --no-build -v minimal + rm -f test-results/dogfood/dogfood-block9-ipc-net8.0.trx test-results/dogfood/dogfood-block9-ipc-net10.0.trx + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net8.0 --filter "FullyQualifiedName~DogfoodBlock9LocalIpcTests" --no-build -v minimal --logger "trx;LogFileName=dogfood-block9-ipc-net8.0.trx" --results-directory test-results/dogfood + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net10.0 --filter "FullyQualifiedName~DogfoodBlock9LocalIpcTests" --no-build -v minimal --logger "trx;LogFileName=dogfood-block9-ipc-net10.0.trx" --results-directory test-results/dogfood + bash scripts/ci/zero-test-guard.sh test-results/dogfood/dogfood-block9-ipc-net8.0.trx test-results/dogfood/dogfood-block9-ipc-net10.0.trx # Dogfood Blocks 1–6 (Phase C validation) dogfood-phase-c: @@ -506,12 +550,18 @@ dogfood-phase-de: # Dogfood Phase F: closed-loop improve on Ashlar dogfood-closedloop: dotnet build src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -v minimal - dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj --filter "FullyQualifiedName~DogfoodClosedLoopTests" --no-build -v minimal + rm -f test-results/dogfood/dogfood-closedloop-net8.0.trx test-results/dogfood/dogfood-closedloop-net10.0.trx + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net8.0 --filter "FullyQualifiedName~DogfoodClosedLoopTests" --no-build -v minimal --logger "trx;LogFileName=dogfood-closedloop-net8.0.trx" --results-directory test-results/dogfood + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net10.0 --filter "FullyQualifiedName~DogfoodClosedLoopTests" --no-build -v minimal --logger "trx;LogFileName=dogfood-closedloop-net10.0.trx" --results-directory test-results/dogfood + bash scripts/ci/zero-test-guard.sh test-results/dogfood/dogfood-closedloop-net8.0.trx test-results/dogfood/dogfood-closedloop-net10.0.trx # Phase F: Continuous self-improvement loop (changelog, test failure store) dogfood-phasef: dotnet build src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -v minimal - dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj --filter "FullyQualifiedName~DogfoodPhaseFTests" --no-build -v minimal + rm -f test-results/dogfood/dogfood-phasef-net8.0.trx test-results/dogfood/dogfood-phasef-net10.0.trx + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net8.0 --filter "FullyQualifiedName~DogfoodPhaseFTests" --no-build -v minimal --logger "trx;LogFileName=dogfood-phasef-net8.0.trx" --results-directory test-results/dogfood + dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net10.0 --filter "FullyQualifiedName~DogfoodPhaseFTests" --no-build -v minimal --logger "trx;LogFileName=dogfood-phasef-net10.0.trx" --results-directory test-results/dogfood + bash scripts/ci/zero-test-guard.sh test-results/dogfood/dogfood-phasef-net8.0.trx test-results/dogfood/dogfood-phasef-net10.0.trx # All dogfood blocks (1–9) + Phase F closed-loop + Phase F dogfood-all: diff --git a/_handoff/readiness/CLOSING-PLAN.md b/_handoff/readiness/CLOSING-PLAN.md index 1bba50568..2ffd1af92 100644 --- a/_handoff/readiness/CLOSING-PLAN.md +++ b/_handoff/readiness/CLOSING-PLAN.md @@ -142,8 +142,9 @@ none, so CI is the first compiler) · **[agent / human / CI]** who performs it. > (runs 34687917934 and 34737292726). No CI behaviour changed. Note also that > `TestOwnershipConventionTests` never reads the `expires` column of a non-`UNOWNED` row, so flipping a > row removes its dated pressure entirely and replaces it with nothing machine-enforced. The live debt -> those three rows now carry is routing: none appears in either readiness path list, so a PR touching -> only them still runs none of their tests. +> those three rows carried was routing: none appeared in either readiness path list, so a PR touching +> only them ran none of their tests. That closed on 2026-09-30: all three are in both lists, with the +> rest of the discovered suites' build closure. --- diff --git a/application/src/Ashlar.Tests.CLI/Tests/Formatting/ValidationResultRenderingTests.cs b/application/src/Ashlar.Tests.CLI/Tests/Formatting/ValidationResultRenderingTests.cs index 85a0a40c5..29654d719 100644 --- a/application/src/Ashlar.Tests.CLI/Tests/Formatting/ValidationResultRenderingTests.cs +++ b/application/src/Ashlar.Tests.CLI/Tests/Formatting/ValidationResultRenderingTests.cs @@ -1,5 +1,6 @@ using Ashlar.CLI.Formatting; using Ashlar.Core.Application.Validation.Models; +using Ashlar.Infrastructure.Validation.Adapters; using FluentAssertions; using Xunit; @@ -11,9 +12,29 @@ public sealed class ValidationConsoleCollection; [Collection("ValidationConsole")] public sealed class ValidationResultRenderingTests { + // Each message below is one the validation adapter actually composes, which + // Every_documented_message_is_one_the_adapter_composes pins, so these pairings cannot drift + // from what `ashlar validate` prints. Since 2026-09-30 an UNFILTERED sweep fails a discovered + // project that executed no test, so a passing "no tests selected" result is reachable only + // under a caller's filter. + private const string EvidenceFailure = + "Validation failed (0/0 tests passed, but 1 project produced no passing evidence); " + + "EvidenceTests.csproj: test results are missing, unreadable or inconsistent; " + + "the run cannot be reported as passing."; + + private const string UnfilteredZeroExecution = + "Validation failed (0/0 tests passed, but 1 project produced no passing evidence); " + + "StressOnlyTests.csproj: executed no tests in an unfiltered sweep (it selected none, and " + + "validate always excludes Category=Stress and Category=DockerOptional); a discovered test " + + "project that runs nothing is not evidence of passing."; + + private const string FilteredEmptySelection = + "Validation passed (0/0 tests); no tests selected: FilteredTests.csproj"; + [Theory] - [InlineData(false, "Validation failed (0/0 tests failed); EvidenceTests.csproj: test results are missing, unreadable or inconsistent")] - [InlineData(true, "Validation passed (0/0 tests); no tests selected: FilteredTests.csproj")] + [InlineData(false, EvidenceFailure)] + [InlineData(false, UnfilteredZeroExecution)] + [InlineData(true, FilteredEmptySelection)] public void Human_output_preserves_project_evidence_and_empty_selection_reasons(bool passed, string message) { var previousOut = Console.Out; @@ -38,4 +59,27 @@ public void Human_output_preserves_project_evidence_and_empty_selection_reasons( Console.SetError(previousError); } } + + [Fact] + public void Every_documented_message_is_one_the_adapter_composes() + { + ValidationServiceAdapter.DescribeOutcome( + false, 0, 0, 0, 0, + new[] { ValidationServiceAdapter.InvalidEvidenceError("EvidenceTests.csproj") }, + Array.Empty()) + .Should().Be(EvidenceFailure); + + var zero = ValidationServiceAdapter.ZeroExecutionError( + "StressOnlyTests.csproj", filter: null, executed: 0, skipped: 0); + zero.Should().NotBeNull("an unfiltered sweep fails a project that executed nothing"); + ValidationServiceAdapter.DescribeOutcome(false, 0, 0, 0, 0, new[] { zero! }, Array.Empty()) + .Should().Be(UnfilteredZeroExecution); + + ValidationServiceAdapter.ZeroExecutionError( + "FilteredTests.csproj", filter: "Category=Smoke", executed: 0, skipped: 0) + .Should().BeNull("only a caller's filter can still produce a passing empty selection"); + ValidationServiceAdapter.DescribeOutcome( + true, 0, 0, 0, 0, Array.Empty(), new[] { "FilteredTests.csproj" }) + .Should().Be(FilteredEmptySelection); + } } diff --git a/ci/cert-gate-assertions.md b/ci/cert-gate-assertions.md index b994c0ea4..447cfd334 100644 --- a/ci/cert-gate-assertions.md +++ b/ci/cert-gate-assertions.md @@ -33,6 +33,7 @@ away, can see what they would be turning off. | **Commercial test roots remain discovered and cover their production graph.** The three receipt roots agree with actual test projects and validation's framework selection; their unconditional commercial references reach all eight commercial projects. | `CommercialCoverageConventionTests`, `CommercialReceiptRoutingTests` | Empty discovery, stale roots, missing references and orphan production projects cannot pass as coverage. Native readiness separately checks nonempty successful TRXs, counters and project/TFM associations; required shell-lint runs its file controls. This is an execution-receipt lens, not full test-selection completeness. | | **Every test project is registered.** Every csproj containing `Microsoft.NET.Test.Sdk` has exactly one row in `ci/test-ownership.tsv`; every row points at a project that still exists; no `UNOWNED` row is past its expiry. | `TestOwnershipConventionTests` | A test project no gate runs re-enters the repo silently. This is exactly how `Ashlar.Commercial.Tests.Fleet.Host` failed for ten days across twenty consecutive runs with no pull request ever running it. | | **The runner column is checked syntax over a real gate vocabulary.** Every registry row has four non-empty tab-separated fields and a unique project; every runner is either `UNOWNED` or a name this repository actually defines as a workflow, a script or a Make target; and a row that names a gate carries `-` for expires, because the expiry column is read only for `UNOWNED` rows. **Scope, stated so it is not overquoted: this proves a runner name DENOTES something, not that the named gate runs that project, runs it unfiltered, or runs it on a pull request.** It catches typos, invented names and renamed-or-deleted gates. None of the five rows corrected on 2026-09-13 would have failed it — three said `UNOWNED`, two named real gates that run only a filtered subset. | `TestOwnershipRunnerConventionTests` | The column was inert text: `banana`, an empty string or a gate deleted two years ago all passed, and the file header said so itself. Five silent holes in the registry parser close with it: a short row is dropped rather than reported; a duplicate project is a last-wins overwrite; a trailing tab collapses a four-field row into three, after which the project reads as unregistered; a tab typed inside a note truncates the row; and a date on a gated row is never read. | +| **Readiness re-runs for everything the suites it runs are built from.** `validate`'s own discovery predicate (`ValidationServiceAdapter.IsDiscoverableTestProject`) selects the test projects the native readiness lanes run through `ci verify`. Their transitive ProjectReference closure - parsed as XML, so attribute order cannot hide an edge, with analyzer-only (`OutputItemType="Analyzer"`, `ReferenceOutputAssembly="false"`) and conditional references followed, and read from every shared MSBuild file a closure project is evaluated from (the implicit `Directory.Build.props`/`.targets` and `Directory.Packages.props` above it, and any file it reaches by an explicit ``) as well as from the project file - must be covered project directory by project directory in BOTH readiness lists, and so must every implicit `Directory.Build.props`/`.targets`/`.rsp`, `Directory.Packages.props`, `.editorconfig`, `.globalconfig`, `nuget.config` or `global.json` above a closure project, and every path outside the project's own directory that a closure project or one of those shared MSBuild files names: in the Include of an item of ANY type except the identity types (a package, framework, assembly or namespace name and the like) and a `None` item that is not copied, in any item's `HintPath`, or in an ``. The item types are a denylist, not an allowlist, so a path-bearing type nobody listed (`Protobuf`, `EditorConfigFiles`, a code-generation input) is still read. A failure names the path, what names it, the list and a suite that needs it. Stated floors (20 suites, 60 closure projects, 6 implicit and 3 cross-directory inputs; measured 22, 70, 10 and 6 on 876cc527) stop an emptied walk from passing, and fixture facts pin the reordered-attribute, analyzer-only, conditional and transitive edges, an edge declared in a shared MSBuild file, and a cross-directory path on a `Protobuf`, `EditorConfigFiles`, `GlobalAnalyzerConfigFiles` or custom item and in a `HintPath`, and refuse `$(Property)`, wildcard, absolute, escaping and missing references and computed item paths and HintPaths - and, in a shared file, a reference or an item path with a directory that is not anchored by `$(MSBuildThisFileDirectory)`, or a wildcard import - rather than approximating them. | `ReadinessClosureConventionTests` | The readiness platform lanes are diff-conditional, so a directory missing from the lists means editing the code a suite tests skips the only lanes that run that suite while editing its tests re-runs them. On 876cc527 that was 37 project directories of a 70-project closure (the kernel suite's four were routed by #684 alone), plus `src/Ashlar.Compat` - compiled into most of the closure through `src/Directory.Build.props` and `Directory.Build.targets` - three nested `Directory.Build.props`, `.editorconfig`, and the brick template and physical-atom-cert sample trees two suites embed or copy. **Routing only:** this does not prove the lanes pass, that a suite selects any test (a skipped lane still counts as a pass in `Readiness summary`), anything about runtime file reads, files read through an MSBuild property function (`$([System.IO.File]::ReadAllText(...))`, which is how the root `Directory.Build.props` and `Directory.Build.targets` read `VERSION` - in both lists by hand, not proven here), paths in item metadata other than `HintPath` (a Protobuf `ProtoRoot` or `AdditionalImportDirs`) or files a compiled input reaches on its own (a `.proto` import), projects an `` task builds inside a target, or projects `ci verify` builds that are not discovered suites. | | **Composition order cannot decide durability, and a host-supplied signer keys both lanes.** `AddCertificationGate` and `AddCertificationInfrastructure` resolve to a durable store in either composition order; a host-supplied signer survives. | `CertificationStoreCompositionTests` | Admissions silently revert to in-memory, and for the CLI — a fresh process per invocation — nothing certified can ever be admitted again. | | **An operator’s signing key reaches the composition lane, and no key material crosses the boundary to get there.** A `CertificationRecordSigner` holding a real key, injected into the composition signer, is the key composition records are minted under — asserted against an independently computed HMAC, with a negative control against the committed dev constant. The composition signer holds no key bytes while delegating, and the internal MAC oracle that makes this possible keeps exactly one declaration and one call site. The gate also warns when its two signers were built independently and are not both on the committed dev key — reference identity, never a key comparison, warning and never refusing. The shipped registration is resolved out of a real container and asserted SILENT, so the claim the design rests on is executed rather than asserted; and the one correct configuration this check false-positives on — two signers holding the same explicit key — is pinned as a false positive, so it cannot be quietly "fixed" into a key comparison. Closes limitation 9. | `CompositionSignerKeyThreadingTests`, `CompositionSignerKeyPathConventionTests` | A composition certificate is supposed to attest the chain its constituents were signed into. Before this, a host that supplied a real key got brick records under that key and composition records under a constant published in the repository, and nothing said so. The convention facts exist because the functional ones all stay green if the delegation becomes decorative. | | **The schema floor refuses a downgraded record.** A forged record with a rewritten gate name verifies at floor 0 and is refused at floor 2. | `SchemaVersionFloorTests` | A record can claim to have passed gates it never ran: the legacy payload lane leaves `Gate`, `GatesPassed`, `Inputs`, `Proposer`, `Attempts` and `Ed25519PublicKey` outside the signed bytes. | diff --git a/ci/test-ownership.tsv b/ci/test-ownership.tsv index 4cb594de3..f7deea363 100644 --- a/ci/test-ownership.tsv +++ b/ci/test-ownership.tsv @@ -44,18 +44,18 @@ commercial/tests/Ashlar.Commercial.Tests.Fleet/Ashlar.Commercial.Tests.Fleet.csp commercial/tests/Ashlar.Commercial.Tests.MeshDirector/Ashlar.Commercial.Tests.MeshDirector.csproj full-platform-readiness-gate - native ci verify -> validate (net8.0 Debug); receipts and graph checked; see docs/CommercialCiCoverage.md samples/hello-brick/HelloBrick.Tests/HelloBrick.Tests.csproj full-platform-readiness-gate - sample; reached by ashlar validate, not by any solution samples/templates/brick/__BrickName__Brick.Tests/__BrickName__Brick.Tests.csproj UNOWNED 2027-06-30 scaffolding template with placeholder identifiers, not a real suite; deliberately the longest expiry and staggered clear of the 2027-03-31 group -src/Ashlar.Analyzers.Tests/Ashlar.Analyzers.Tests.csproj full-platform-readiness-gate - native ci verify -> validate (net8.0 Debug) discovers and runs it. The prior note claimed these tests were never visible to any PR or master check; both halves are refuted by separate runs. MASTER: run 34737292726 (push, green) and run 34687917934 (push, the #605 commit that added the AnalyzerReferenceSet guards; that run is RED overall for an unrelated FileBarrierAuditSinkTests failure, and its sweep job is green and lists all seven SelfRecursiveRegistrationAnalyzerTests). PULL REQUEST: run 34686265374 (pull_request, green, 4943/4943, same seven tests). NOT run by its own changes: src/Ashlar.Analyzers.Tests/** is in neither readiness path list, so a PR touching only THIS TEST PROJECT skips the heavy lanes. Note since 2026-09-30 src/Ashlar.Analyzers/** (the analyzer itself) IS in both lists, added for the kernel suite's build closure - and it does not match src/Ashlar.Analyzers.Tests/, verified under both the bash and GitHub matchers, so this sentence is still literally true. The sweep prints no per-project count and a zero-selection project still passes (ValidationServiceAdapter.cs:248): a lane, not a receipt. [corrected 2026-09-13] -src/Ashlar.Ingress.AwsSns.Tests/Ashlar.Ingress.AwsSns.Tests.csproj full-platform-readiness-gate - native ci verify -> validate (net8.0 Debug) discovers and runs it; observed in readiness run 34737292726. NOT run by its own changes: src/Ashlar.Ingress.AwsSns.Tests/** is in neither readiness path list, so an ingress-only PR skips the heavy lanes. The sweep prints no per-project count and a zero-selection project still passes (ValidationServiceAdapter.cs:248): a lane, not a receipt. [corrected 2026-09-13: previously UNOWNED with "no gate runs it"] -src/Ashlar.Ingress.DynamoDb.Tests/Ashlar.Ingress.DynamoDb.Tests.csproj full-platform-readiness-gate - native ci verify -> validate (net8.0 Debug) discovers and runs it; observed in readiness run 34737292726. NOT run by its own changes: src/Ashlar.Ingress.DynamoDb.Tests/** is in neither readiness path list, so an ingress-only PR skips the heavy lanes. The sweep prints no per-project count and a zero-selection project still passes (ValidationServiceAdapter.cs:248): a lane, not a receipt. [corrected 2026-09-13: previously UNOWNED with "no gate runs it"] +src/Ashlar.Analyzers.Tests/Ashlar.Analyzers.Tests.csproj full-platform-readiness-gate - native ci verify -> validate (net8.0 Debug) discovers and runs it. The prior note claimed these tests were never visible to any PR or master check; both halves are refuted by separate runs. MASTER: run 34737292726 (push, green) and run 34687917934 (push, the #605 commit that added the AnalyzerReferenceSet guards; that run is RED overall for an unrelated FileBarrierAuditSinkTests failure, and its sweep job is green and lists all seven SelfRecursiveRegistrationAnalyzerTests). PULL REQUEST: run 34686265374 (pull_request, green, 4943/4943, same seven tests). Run by its own changes as of 2026-09-30: src/Ashlar.Analyzers.Tests/** is in BOTH readiness path lists, added with the rest of the discovered suites' build closure, so a PR touching only this test project re-runs the heavy lanes. src/Ashlar.Analyzers/** (the analyzer itself) is a separate entry in both lists, added earlier for the kernel suite's build closure; it does not match src/Ashlar.Analyzers.Tests/ under either the bash or the GitHub matcher, which is why the test directory needed its own entry. The sweep prints no per-project count. Since 2026-09-30 an unfiltered sweep fails a discovered project that executes zero tests (ValidationServiceAdapter.ZeroExecutionError), but that floor is one executed test, not this project's population: a lane, not a receipt. [corrected 2026-09-13] +src/Ashlar.Ingress.AwsSns.Tests/Ashlar.Ingress.AwsSns.Tests.csproj full-platform-readiness-gate - native ci verify -> validate (net8.0 Debug) discovers and runs it; observed in readiness run 34737292726. Run by its own changes as of 2026-09-30: src/Ashlar.Ingress.AwsSns.Tests/** and the project it tests, src/Ashlar.Ingress.AwsSns/**, are both in BOTH readiness path lists, added with the rest of the discovered suites' build closure, so an ingress-only PR re-runs the heavy lanes. The sweep prints no per-project count. Since 2026-09-30 an unfiltered sweep fails a discovered project that executes zero tests (ValidationServiceAdapter.ZeroExecutionError), but that floor is one executed test, not this project's population: a lane, not a receipt. [corrected 2026-09-13: previously UNOWNED with "no gate runs it"] +src/Ashlar.Ingress.DynamoDb.Tests/Ashlar.Ingress.DynamoDb.Tests.csproj full-platform-readiness-gate - native ci verify -> validate (net8.0 Debug) discovers and runs it; observed in readiness run 34737292726. Run by its own changes as of 2026-09-30: src/Ashlar.Ingress.DynamoDb.Tests/** and the project it tests, src/Ashlar.Ingress.DynamoDb/**, are both in BOTH readiness path lists, added with the rest of the discovered suites' build closure, so an ingress-only PR re-runs the heavy lanes. The sweep prints no per-project count. Since 2026-09-30 an unfiltered sweep fails a discovered project that executes zero tests (ValidationServiceAdapter.ZeroExecutionError), but that floor is one executed test, not this project's population: a lane, not a receipt. [corrected 2026-09-13: previously UNOWNED with "no gate runs it"] src/Ashlar.Mcp.Client.Tests/Ashlar.Mcp.Client.Tests.csproj mcp-a2a-gate - MCP client src/Ashlar.Mcp.Server.Tests/Ashlar.Mcp.Server.Tests.csproj mcp-a2a-gate - MCP server src/Ashlar.Tests.AI.Pipeline/Ashlar.Tests.AI.Pipeline.csproj kernel-gate;full-platform-readiness-gate - Kernel Tier A -> make meai-pipeline-gate (net8.0 Release); native readiness -> ci verify -> validate (net8.0) src/Ashlar.Tests.Application/Ashlar.Tests.Application.csproj kernel-coverage-gate - application layer -src/Ashlar.Tests.BackgroundAgents/Ashlar.Tests.BackgroundAgents.csproj perf-gate-tier-a;full-platform-readiness-gate - background agents. perf-gate-tier-a selects only FullyQualifiedName~Ashlar.Tests.BackgroundAgents.Performance (scripts/perf-gate-tier-a.sh:23-24); the rest of the project, including the Autonomy suites, runs unfiltered in readiness native ci verify -> validate, observed in run 34737292726. src/Ashlar.Tests.BackgroundAgents/** IS in both readiness path lists, so changes here do route to the heavy lanes. [corrected 2026-09-13: the row named only perf-gate-tier-a, which runs one namespace of it] -src/Ashlar.Tests.Contracts/Ashlar.Tests.Contracts.csproj products-gate;full-platform-readiness-gate - products-gate runs DistributedContractTests only; the other seven contract suites run unfiltered in readiness native ci verify -> validate on master, observed in run 34737292726. NOT run by their own changes: src/Ashlar.Tests.Contracts/** is in neither readiness path list, and products-gate triggers only on src/Ashlar.Tests.Contracts/DistributedContractTests.cs, so a PR editing another contract suite runs none of them. [corrected 2026-09-13: the note said they had no gate; readiness runs them on master] +src/Ashlar.Tests.BackgroundAgents/Ashlar.Tests.BackgroundAgents.csproj perf-gate-tier-a;full-platform-readiness-gate - background agents. perf-gate-tier-a selects only FullyQualifiedName~Ashlar.Tests.BackgroundAgents.Performance (scripts/perf-gate-tier-a.sh, its second dotnet test invocation); the rest of the project, including the Autonomy suites, runs unfiltered in readiness native ci verify -> validate, observed in run 34737292726. src/Ashlar.Tests.BackgroundAgents/** IS in both readiness path lists, so changes here do route to the heavy lanes. [corrected 2026-09-13: the row named only perf-gate-tier-a, which runs one namespace of it] +src/Ashlar.Tests.Contracts/Ashlar.Tests.Contracts.csproj products-gate;full-platform-readiness-gate - products-gate runs DistributedContractTests only; the other seven contract suites run unfiltered in readiness native ci verify -> validate on master, observed in run 34737292726. Run by their own changes as of 2026-09-30: src/Ashlar.Tests.Contracts/** is in BOTH readiness path lists, added with the rest of the discovered suites' build closure, so a PR editing any contract suite re-runs the readiness lanes, which run the project unfiltered; products-gate still triggers only on src/Ashlar.Tests.Contracts/DistributedContractTests.cs. [corrected 2026-09-13: the note said they had no gate; readiness runs them on master] src/Ashlar.Tests.Domain/Ashlar.Tests.Domain.csproj kernel-coverage-gate - domain, 100% line coverage target src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj cert-gate - carries the required check -src/Ashlar.Tests.Kernel/Ashlar.Tests.Kernel.csproj full-platform-readiness-gate - native ci verify -> validate (net8.0 Debug) discovers and runs it on all three native lanes; observed green in readiness run 36639604217 (push, master f5846d3a), 412 cases per lane, and reproduced locally 7/7 at 412/412 in the devtest container. Run by changes to the code it guards, as of 2026-09-30: src/Ashlar.Manifest/**, src/Ashlar.Policies.Dev/**, src/Ashlar.Certification.Contracts/** and src/Ashlar.Analyzers/** were added to BOTH readiness path lists alongside src/Ashlar.Tests.Kernel/**, so editing GateStore.cs or AdmissionGate.cs now re-runs these 412 cases; before that it did not. Those four are the kernel project's transitive ProjectReference closure that no entry covered; the wider absent set (41 dirs across all 22 discovered suites) is still open. The sweep prints no per-project count and a zero-selection project still passes (ValidationServiceAdapter.cs:248), and readiness-summary counts a skipped lane as a pass: a lane, not a receipt. [corrected 2026-09-29: previously claimed kernel-gate, which lists this project in neither path list and builds Ashlar.Runtime.sln, where it does not appear] +src/Ashlar.Tests.Kernel/Ashlar.Tests.Kernel.csproj full-platform-readiness-gate - native ci verify -> validate (net8.0 Debug) discovers and runs it on all three native lanes; observed green in readiness run 36639604217 (push, master f5846d3a), 412 cases per lane, and reproduced locally 7/7 at 412/412 in the devtest container. Run by changes to the code it guards, as of 2026-09-30: src/Ashlar.Manifest/**, src/Ashlar.Policies.Dev/**, src/Ashlar.Certification.Contracts/** and src/Ashlar.Analyzers/** were added to BOTH readiness path lists alongside src/Ashlar.Tests.Kernel/**, so editing GateStore.cs or AdmissionGate.cs now re-runs these 412 cases; before that it did not. Those four are the kernel project's transitive ProjectReference closure that no entry covered; the wider absent set (41 dirs across all 22 discovered suites, these four included) is closed as of 2026-09-30: the other 37 directories, plus the closure's implicit and cross-directory build inputs, are in BOTH readiness path lists, and ReadinessClosureConventionTests (cert-gate) recomputes the closure from the tree, including references declared in shared MSBuild files, and fails naming any directory or input that either list misses. It does not trace files read through MSBuild property functions. The sweep prints no per-project count; since 2026-09-30 an unfiltered sweep fails a discovered project that executes zero tests (ValidationServiceAdapter.ZeroExecutionError), but that floor is one executed test, not these 412, and readiness-summary counts a skipped lane as a pass: a lane, not a receipt. [corrected 2026-09-29: previously claimed kernel-gate, which lists this project in neither path list and builds Ashlar.Runtime.sln, where it does not appear] src/Ashlar.Tests.Orchestration/Ashlar.Tests.Orchestration.csproj perf-gate-tier-a - orchestration src/Ashlar.Tests.Transport/Ashlar.Tests.Transport.csproj kernel-gate-tier-c - transport src/Ashlar.Transport.A2A.Server.Tests/Ashlar.Transport.A2A.Server.Tests.csproj mcp-a2a-gate - A2A server diff --git a/docs/CiGateInventory.md b/docs/CiGateInventory.md index dee36ad63..1c61ec7a6 100644 --- a/docs/CiGateInventory.md +++ b/docs/CiGateInventory.md @@ -28,11 +28,15 @@ Native readiness's `ci verify` also calls root `validate`, which discovers, buil commercial ones** (corrected 2026-09-13; run 34737292726 logs `Found 22 test project(s)` and `Tests: 5245/5245`). `ci/test-ownership.tsv` carries 23 registered test projects; the sweep reaches 22 of them, the difference being the `__BrickName__` scaffolding template, which `ValidationServiceAdapter` excludes by design (`templates` directory and `__Placeholder__` token). The discovered count is not pinned by anything and has drifted with the tree (26 on 2026-08-31, 23 on 2026-09-12, 22 on 2026-09-13), so it is an observation per run, not a floor. Among them are three projects the ownership registry recorded as UNOWNED until that correction: `Ashlar.Analyzers.Tests`, `Ashlar.Ingress.AwsSns.Tests` and -`Ashlar.Ingress.DynamoDb.Tests`. The sweep emits no per-project counts, and a project that selects -zero tests still passes, so this is a lane and not a receipt. The three commercial suites — Fleet and +`Ashlar.Ingress.DynamoDb.Tests`. The sweep emits no per-project counts; since 2026-09-30 an unfiltered +sweep fails a project that executes zero tests (`ValidationServiceAdapter.ZeroExecutionError`), but +that floor is one test, so this is still a lane and not a receipt. The three commercial suites — Fleet and MeshDirector on net8.0, Fleet.Host on net10.0 — are the only ones with a receipt check. Its post-sweep receipt check and uploads are described in [Commercial CI coverage](CommercialCiCoverage.md). -This indirect route reaches projects outside the solution and workflow path literals. +This indirect route reaches projects outside the solution and workflow path literals. Because the +lanes that run it are diff-conditional, the readiness path lists cover the sweep's whole build +closure rather than the test directories alone; `ReadinessClosureConventionTests` recomputes that +closure from the tree with the same discovery predicate. ### Checks that are safe to require (always report on PRs) @@ -109,7 +113,7 @@ Six workflows carry a `schedule`: `distribution-matrix-gate` (Mon 10:00 UTC), `f | `shell-lint.yml` | Shell lint / `shell-lint` | **every PR** (no paths) — **required** | dispatch — always reports; cheap parse/lint check | | `docs-link-check.yml` | Docs Link Check / `lychee (README + docs)` | **every PR** (no paths) — **required** | push, dispatch — always reports; ~30s lychee run | | `layer-boundary.yml` | layer-boundary / `verify` | every PR (`paths: "**"`, types opened/synchronize/reopened/edited) | — | -| `full-platform-readiness-gate.yml` | Full Platform Readiness Gate / `changes`, platform lanes, `Readiness summary` | **every PR** (no paths; types opened/synchronize/reopened/ready_for_review) — `changes` job runs the heavy lanes only when a core path is in the diff (Dockerfiles, setup/install scripts, spine sources, CLI + CLI tests, `commercial/**`, Orchestration/Kernel test projects, the kernel suite's own dependency closure (`Ashlar.Manifest`, `Ashlar.Policies.Dev`, `Ashlar.Certification.Contracts`, `Ashlar.Analyzers`), StableSdkHostSample); otherwise `Readiness summary` passes in ~1 min — **required** (`Readiness summary`). Production images (`docker-all-images`) never build on PRs. | push `master`/`main`/`cursor/**` (same path list), **weekly schedule**, dispatch | +| `full-platform-readiness-gate.yml` | Full Platform Readiness Gate / `changes`, platform lanes, `Readiness summary` | **every PR** (no paths; types opened/synchronize/reopened/ready_for_review) — `changes` job runs the heavy lanes only when a core path is in the diff (Dockerfiles, setup/install scripts, the CLI, `commercial/**`, StableSdkHostSample, the shared root build inputs, and the whole build closure of every test project `validate` discovers: each project directory in their transitive ProjectReference closure, the nested `Directory.Build.props` and `.editorconfig` that closure imports implicitly, and every path outside its own directories that its items (of any type but package, framework and similar identities), HintPaths and imports name - `src/Ashlar.Compat/**`, `samples/templates/brick/**`, `samples/physical-atom-cert/**`; `ReadinessClosureConventionTests` recomputes that closure in cert-gate and fails naming anything neither list covers); otherwise `Readiness summary` passes in ~1 min — **required** (`Readiness summary`). Production images (`docker-all-images`) never build on PRs. | push `master`/`main`/`cursor/**` (same path list), **weekly schedule**, dispatch | | `uat-gate.yml` | UAT / `uat (tiers 0-2, 4-10)`, `uat cross-platform (tiers 8, 9)` | **every PR** (no paths — deliberate, see file header) | push `master`, dispatch | | `application-gate.yml` | Application Gate / `application-gate` | paths: `application/**`, VirtualProduction tests, `scripts/application-gate*.sh`, `scripts/prod-dry-run.sh`, `Makefile`, … | dispatch | | `dependency-boundary.yml` | dependency-boundary / `verify` | paths: `**/*.csproj`, `commercial/**`, `application/**`, `src/**`, `LICENSING.md`, boundary scripts | push, dispatch | diff --git a/docs/HowGatesGoQuiet.md b/docs/HowGatesGoQuiet.md index 624bc5b9a..037be70cc 100644 --- a/docs/HowGatesGoQuiet.md +++ b/docs/HowGatesGoQuiet.md @@ -281,15 +281,16 @@ give the derivation a positive control with a floor, so a population that collap fails instead of passing. Where you genuinely cannot derive it, say what the lens excludes *in the test*, next to the assertion, so the next reader is not told a narrow fact in wide words. -**Open, in this shape, right now.** Deriving the lane population turned up three lanes whose -`--filter` selects **zero** tests in the project they run, so each passes having executed nothing: -`scripts/security-gate-tier-b.sh:11` and `scripts/application-gate-tier-c.sh:11` both run -`-f net8.0` while naming classes under `Tests/API` and `Tests/VirtualProduction`, which the csproj -compiles out on net8.0; and `scripts/compat-gate-tier-a.sh:11` names -`MeshTaskExecutionServiceTests`, which exists in no project — the real tests are -`MeshTaskExecutionServiceGapCoverageTests` in `Ashlar.Commercial.Tests.Fleet`. Not fixed here: -each needs a decision about which framework or project the step should have been running, and that -is a separate change from the window work these were found by. +**Closed 2026-09-30, and first recorded here as open.** Deriving the lane population turned up +three lanes whose `--filter` selected **zero** tests in the project they ran, so each passed having +executed nothing: security-gate Tier B and application-gate Tier C both ran `-f net8.0` while naming +classes the csproj compiles for net10.0 only; and compat-gate Tier A step 1 named +`MeshTaskExecutionServiceTests` in `Ashlar.Tests.Infrastructure`, although the class had moved to +`Ashlar.Commercial.Tests.Fleet` in #151. (An earlier version of this paragraph said the class "exists +in no project"; it exists, in the commercial Fleet suite.) All three now select their intended tests +(44, 4 and 1), and `scripts/ci/zero-test-guard.sh` follows every pinned filter so that a selection of +zero fails its lane - with `scripts/ci/verify-zero-test-guard-wiring.py`, in required `shell-lint`, +refusing a pinned filter that has no guard. ## 13. The guard is written, it compiles, and it cannot fire @@ -505,5 +506,7 @@ placement decision, so correcting the registry meant correcting a merged design **A pessimistic record is not the safe direction.** Both instances understated coverage, and the repository treated understatement as conservative. It is not: it invited work to close a gap that did not exist, and it sat in the file whose purpose is truthful recording. The real gap here was -narrower and went unnamed — those projects appear in neither readiness path list, so a PR touching -only them still runs none of their tests. +narrower and went unnamed — those projects appeared in neither readiness path list, so a PR touching +only them ran none of their tests. That gap closed on 2026-09-30, when all three were added to both +lists with the rest of the discovered suites' build closure, which ReadinessClosureConventionTests +now gates. diff --git a/docs/MeshPhase6LeasesAndCheckpoints.md b/docs/MeshPhase6LeasesAndCheckpoints.md index 05cf544c0..a46693e37 100644 --- a/docs/MeshPhase6LeasesAndCheckpoints.md +++ b/docs/MeshPhase6LeasesAndCheckpoints.md @@ -49,7 +49,8 @@ See **`docs/Configuration.md`**. ## Tests -- `Ashlar.Tests.Infrastructure` → `Tests/Fleet/MeshTaskPlacementServiceTests.cs`, `MeshTaskExecutionServiceTests.cs` +- `Ashlar.Tests.Infrastructure` → `Tests/Fleet/MeshTaskPlacementServiceTests.cs` +- `Ashlar.Commercial.Tests.Fleet` → `MeshTaskExecutionServiceTests.cs` (moved there in #151; corrected 2026-09-30) ## Revision history diff --git a/docs/RELEASE_RUNBOOK.md b/docs/RELEASE_RUNBOOK.md index b820b3d9b..b5bb52f67 100644 --- a/docs/RELEASE_RUNBOOK.md +++ b/docs/RELEASE_RUNBOOK.md @@ -88,6 +88,81 @@ Trusted Publishing: register **`release.yml`** and **`release-nuget.yml`** as ne Skip this and the tag is the first time the post-push chain runs against a live feed, on the one trigger where a failure cannot be taken back. The staging-feed path does not substitute: the post-push scripts target nuget.org, and `NUGET_STAGING_FEED_URL` is unset. +9. **Confirm the readiness gate verified the exact commit you are tagging — read the verdict, not + the tick.** `Readiness summary` is required, so it has to pass when its platform lanes are + skipped, and its green tick cannot tell "every platform passed" from "no platform ran". This is + the last step because steps 4–7 change the release commit; check the SHA you will actually tag: + + ```bash + git fetch origin + SHA=$(git rev-parse origin/master) # or whichever commit you will tag - in full + bash scripts/release/readiness-verdict-for-sha.sh "$SHA" + ``` + + It reads the `Readiness verdict` annotation that each `Readiness summary` check run carries + (read-only; needs `gh auth login`), prints every run it found, and exits 0 only on a `VERIFIED` + read from GitHub. The last line is the outcome: + + | last line | means | do | + |---|---|---| + | `VERIFIED`, exit 0 | every lane group ran and passed on this exact commit | tag it. Only this exits 0. A `VERIFIED` line that ends `[offline fixture - not evidence]` exits 3: it read a test fixture, not GitHub, so unset `READINESS_VERDICT_FIXTURE` and run it again | + | `REFUSED (no-readiness-run)` | nothing ran on this SHA — usually the path filter, below | get evidence, below | + | `REFUSED (not-verified)` | a run exists but every lane was skipped | get evidence, below | + | `REFUSED (partial)` | only a `pull_request`/`merge_group` run: production images never built | get evidence, below | + | `REFUSED (merge-commit-only)` | the only `verified` run is a `pull_request` run, which tested this commit merged into its base, not this commit | get evidence, below | + | `REFUSED (failed)` or `(conflicting-verdicts)` | a lane failed or was cancelled in some run of this SHA; a pass in another run does not outweigh it | open that run; if a newer push cancelled it or it hit a known flake, re-run **that** run with `gh run rerun ` (its latest attempt supersedes the failure), but only as *Re-running a `master` run*, below, allows; otherwise do not tag. A dispatch on a throwaway branch cannot clear this: the failed run stays attached to the SHA, and a `verified` dispatch beside it is `conflicting-verdicts` | + | `REFUSED (in-progress)` | a run of this SHA has not finished and could still fail | wait, run the script again | + | `REFUSED (missing-annotation)` | the run has no verdict annotation: the summary job was cancelled early, or the commit predates the annotation | re-run it, as *Re-running a `master` run*, below, allows; a commit that predates the annotation cannot be certified by the script — read the verdict in the run summary and say so in the release notes | + | `REFUSED (malformed-annotation)`, `(foreign-check-run)`, `(sha-mismatch)` | the evidence is not trustworthy as it stands | stop and investigate; do not tag around it | + | `UNDETERMINED (...)`, exit 2 | the script could not read the evidence (gh unauthenticated, API error, a short SHA) | fix that and run it again — this is not a pass | + + `partial` is refused, with no override. Push, schedule and `workflow_dispatch` runs execute every + lane group, so a partial verdict only ever comes from a pull request or merge-queue run, which + never builds the production Docker images. + + **Re-running a `master` run.** A failed run is the likeliest refusal on a release SHA, not a + corner case: a merge that touches a readiness path cancels the readiness run still in progress + for the commit before it, and that run's `Readiness summary` still runs and records `failed`. + Observed: `master` push run 36637044333 on `f34a9f67` was cancelled 25 minutes in by the push + of the `0.2.0` release commit, and its `Readiness summary` concluded `failure`. The fix is a + re-run of *that* run, and a re-run is not free. It keeps the original run's ref, so a re-run of + a push, schedule or dispatch run of `master` joins `master`'s concurrency group + (`full-platform-readiness-refs/heads/master`, `cancel-in-progress: true`): starting it cancels + any `master` readiness run in flight — which then records `failed` on its own commit — and the + next merge that touches a readiness path cancels the re-run. So re-run only when this shows + nothing queued or in progress, and hold readiness-path merges until the re-run has completed: + + ```bash + gh run list --workflow full-platform-readiness-gate.yml --branch master --limit 5 + gh run rerun + ``` + + The throwaway-branch dispatch below does **not** clear a failed run. The failure stays attached + to the SHA, and a `verified` dispatch beside it is `conflicting-verdicts`; only a later attempt + of the failed run itself supersedes it. + + **The path-filter wrinkle.** On push to `master` the readiness gate is path-filtered: a commit + whose diff touches no readiness path (a docs-only commit, for example) gets **no run at all**, so + there is nothing to read and the script refuses. A release commit usually does get one, because + it bumps `VERSION` and that is a readiness path; a docs fix-up merged after it does not. To get + evidence for such a SHA, dispatch the gate on a branch pinned to it: + + ```bash + # $SHA as above + git push origin "$SHA:refs/heads/readiness-evidence/${SHA:0:12}" + gh workflow run full-platform-readiness-gate.yml --ref "readiness-evidence/${SHA:0:12}" + # ...when that run has completed: + bash scripts/release/readiness-verdict-for-sha.sh "$SHA" + git push origin --delete "readiness-evidence/${SHA:0:12}" + ``` + + A dispatch runs every lane group unconditionally, but `gh workflow run --ref` accepts a branch or + tag, not a SHA, and the run evaluates whatever that ref points at when it starts. Dispatching on + `master` is evidence for the SHA only if `master` has not moved by then, and a `master` dispatch + shares the push runs' concurrency group, so the next merge that starts a readiness run cancels + it. A throwaway branch pinned to the SHA has neither problem. Never use a `v*` tag for this: + that is the release trigger. The script only reads check runs attached to the exact SHA, so a + dispatch that resolved to some other commit can never be counted as evidence for this one. ## After `release.yml` diff --git a/docs/Testing.md b/docs/Testing.md index 4bb93f685..d98d3fc6d 100644 --- a/docs/Testing.md +++ b/docs/Testing.md @@ -200,7 +200,7 @@ No pull-request lane runs `dotnet test Ashlar.sln`. The PR-triggered workflows r ```bash bash scripts/ci/kernel-coverage-gate.sh # kernel-coverage-gate.yml — Domain 100%, Infrastructure -f net10.0 --filter "FullyQualifiedName!~RuntimeStudioBlackBoxSmokeTests&Category!=External" (80% floor), Core.Application 67% -bash scripts/run-cert-gate.sh # cert-gate.yml — Certification + GenerationSafety + AstMutationEngine, -f net8.0, zero-test guard +bash scripts/run-cert-gate.sh # cert-gate.yml — Certification + GenerationSafety + AstMutationEngine, -f net8.0, zero-test guard + skip guard (scripts/cert-gate-skipped.baseline) make kernel-gate # kernel-gate.yml — tier A (tier-b..e / kernel-gate-full also dispatchable) make application-gate-tier-a # application-gate.yml — tier-c = in-process Ashlar.API WebApplicationFactory tests make testing-strategy-gate # testing-strategy-gate.yml — PR diff rules (gap freeze, ProdStyle wiring) @@ -211,7 +211,7 @@ make ci-verify # `ashlar ci verify` — build + C#-dr ### Opt-in external suites (Skipped, not silently Passed) -Tests that need an external dependency use `[OptInFact("", "")]` from `Ashlar.Tests.Infrastructure.Helpers` instead of `if (!enabled) return;`. Without the variable the test shows up as **Skipped** with a reason naming the switch (xunit 2.x evaluates it at discovery time). Host-heavy tests that must not run on GitHub runners use `[NotOnCiFact("")]` (skipped when `CI` / `GITHUB_ACTIONS` is `true`). +Tests that need an external dependency use `[OptInFact("", "")]` from `Ashlar.Tests.Infrastructure.Helpers` instead of `if (!enabled) return;`. Without the variable the test shows up as **Skipped** with a reason naming the switch (xunit 2.x evaluates it at discovery time). Host-heavy tests that must not run on GitHub runners use `[NotOnCiFact("")]` (skipped when `CI` / `GITHUB_ACTIONS` is `true`). A test project made ONLY of such tests fails an unfiltered `ashlar validate` on any lane where they are all skipped, since 2026-09-30, so give it at least one fact that runs without the dependency. | Variable | Enables | Also needs | |----------|---------|------------| diff --git a/docs/certification-evidence.md b/docs/certification-evidence.md index 4a57fafb6..0431f55e8 100644 --- a/docs/certification-evidence.md +++ b/docs/certification-evidence.md @@ -38,7 +38,7 @@ Local cert-gate on integration tip (`9baf34a9`): ``` Test Run Successful. Total tests: 19, Passed: 19 -cert-gate executed 19 tests (expected>=19, derived from --list-tests). +cert-gate reported 19 tests (expected>=19, derived from --list-tests). ``` Dogfood on integration: `HonestCursorGeneration_Admits_WithZeroEscapeRate` PASS, `BuggyCursorGeneration_Rejects` PASS. @@ -646,7 +646,7 @@ redundant guard is exactly what a careful proposer writes. 2. **Composition seam check is TYPE-level only.** The seam validator checks producer/consumer type compatibility (e.g. `string` vs `int`) but not semantic mismatches where types align (e.g. file path vs URL, both `string`). Graph-mutation teeth only partially compensate for this gap. -3. **cert-gate expected count is derived at runtime.** The zero-test guard compares executed count against `dotnet test --list-tests` on the cert-gate filter (see `scripts/cert-gate-config.sh`). If the filter or test discovery breaks, the guard fails closed. +3. **cert-gate expected count is derived at runtime.** The zero-test guard compares the TRX `total` (which counts skipped tests) against `dotnet test --list-tests` on the cert-gate filter (see `scripts/cert-gate-config.sh`), and `scripts/cert-gate-skip-guard.sh` pins `total - executed` to the committed `scripts/cert-gate-skipped.baseline`, so a newly skipped certification test is a reviewed diff rather than a silent pass. If the filter or test discovery breaks, the guards fail closed. 4. **Session containment is opt-in and covers compilation + execution; the host keeps orchestration, mutant compilation, and judgment.** With `BuildCandidateInSession=true`, diff --git a/docs/knowledge-graph.json b/docs/knowledge-graph.json index 578b09ab7..e4054abd5 100644 --- a/docs/knowledge-graph.json +++ b/docs/knowledge-graph.json @@ -15,8 +15,8 @@ "operator_facing_config_variables": 153, "operator_vars_missing_from_configuration_reference": 18, "operator_vars_absent_from_all_documentation": 0, - "test_files_declaring_facts": 730, - "declared_facts": 4380, + "test_files_declaring_facts": 731, + "declared_facts": 4399, "workflows": 62 }, "operator_vars_missing_from_configuration_reference": [ @@ -2849,7 +2849,7 @@ "file": "application/src/Ashlar.Tests.CLI/Tests/Formatting/ValidationResultRenderingTests.cs", "project": "Ashlar.Tests.CLI", "namespace": "Ashlar.Tests.CLI.Tests.Formatting", - "declared_facts": 1, + "declared_facts": 2, "in_cert_gate_filter": false }, { @@ -5197,6 +5197,13 @@ "declared_facts": 6, "in_cert_gate_filter": true }, + { + "file": "src/Ashlar.Tests.Infrastructure/Tests/Certification/ReadinessClosureConventionTests.cs", + "project": "Ashlar.Tests.Infrastructure", + "namespace": "Ashlar.Tests.Infrastructure.Tests.Certification", + "declared_facts": 11, + "in_cert_gate_filter": true + }, { "file": "src/Ashlar.Tests.Infrastructure/Tests/Certification/RealModelCompositionProposerDogfoodTests.cs", "project": "Ashlar.Tests.Infrastructure", @@ -6846,7 +6853,7 @@ "file": "src/Ashlar.Tests.Infrastructure/Tests/Validation/ValidationServiceAdapterGapCoverageTests.cs", "project": "Ashlar.Tests.Infrastructure", "namespace": "Ashlar.Tests.Infrastructure.Tests.Validation", - "declared_facts": 20, + "declared_facts": 27, "in_cert_gate_filter": false }, { diff --git a/docs/knowledge-graph.md b/docs/knowledge-graph.md index df6327f08..b0daaf38b 100644 --- a/docs/knowledge-graph.md +++ b/docs/knowledge-graph.md @@ -19,8 +19,8 @@ remembered to update — staleness here is a red build. | of those, operator-facing (not test-only) | 153 | | missing from `docs/Configuration.md` | 18 | | **mentioned in no document at all** | **0** | -| test files declaring xUnit facts | 730 | -| declared facts | 4380 | +| test files declaring xUnit facts | 731 | +| declared facts | 4399 | | workflows | 62 | ## Namespaces cert-gate selects diff --git a/docs/production-readiness/ApplicationReadiness-v1.md b/docs/production-readiness/ApplicationReadiness-v1.md index 02f31c239..9369c9ceb 100644 --- a/docs/production-readiness/ApplicationReadiness-v1.md +++ b/docs/production-readiness/ApplicationReadiness-v1.md @@ -1,6 +1,10 @@ # Application Readiness v1 **Status: APPLICATION GATE GREEN** (Tiers A–D, 2026-05-19) +[corrected 2026-09-30: Tiers A and D are SUPPORTED, carried forward from CI on PR #110's head +`ee8befa7`, whose tree is identical to this sheet's commit `0ecb5bfc`. Tier B's test step +could reach only one of the four suites it names. No CI run of Tier C was found, and it now +selects no tests. See [Corrections](#corrections-2026-09-30).] Track after [Kernel Readiness v1](KernelReadiness-v1.md). **Plan:** [Application Hardening Plan v1](ApplicationHardeningPlan-v1.md) @@ -14,16 +18,16 @@ make application-gate-full # after make kernel-gate-full (~2–3 min without D | Date | Gate | Result | Notes | |------|------|--------|-------| -| 2026-05-19 | A–D local | **PASS** | agent-server `/health` + `/api/status` | +| 2026-05-19 | A–D local | **PASS** [corrected 2026-09-30: no single run backs this. Evidence per tier is in the table below.] | agent-server `/health` + `/api/status` | ## Tier summary | Tier | Focus | Command | Status | |------|--------|---------|--------| -| A | Product build + CLI validate | `make application-gate-tier-a` | **PASS** | -| B | CLI tests + doctor | `make application-gate-tier-b` | **PASS** | -| C | In-process API HTTP | `make application-gate-tier-c` | **PASS** | -| D | Agent-server Compose | `make application-gate-tier-d` | **PASS** | +| A | Product build + CLI validate | `make application-gate-tier-a` | **PASS** [corrected 2026-09-30: SUPPORTED, carried forward ee8befa7 → 0ecb5bfc (PR #110 head, identical tree f1e22dfd), by Application Gate run 26262666898 (2026-05-22, step Tier A).] | +| B | CLI tests + doctor | `make application-gate-tier-b` | **PASS** [corrected 2026-09-30: UNSUPPORTED-AT-THE-TIME in part. Three of the four suites its filter names had no xUnit test the filter could select. The doctor leg cannot fail the tier. No CI run found.] | +| C | In-process API HTTP | `make application-gate-tier-c` | **PASS** [corrected 2026-09-30: UNEVIDENCED then (4 test methods selectable; no CI run found). LATER-DECAYED: it has selected no tests since 2026-08-17 and still prints PASS.] | +| D | Agent-server Compose | `make application-gate-tier-d` | **PASS** [corrected 2026-09-30: SUPPORTED, carried forward ee8befa7 → 0ecb5bfc (PR #110 head, identical tree f1e22dfd), by Prod dry run (Compose) run 26262666791, "agent-server stack" (2026-05-22).] | ## Next: composition & mesh @@ -35,5 +39,26 @@ See [Composition & mesh readiness v1](CompositionMeshReadiness-v1.md). ## Sign-off -- [x] `application-gate-full` green (2026-05-19) -- [x] Kernel gate green same day +- [x] `application-gate-full` green (2026-05-19) [corrected 2026-09-30: see the tier table. B and C + are not backed.] +- [x] Kernel gate green same day [corrected 2026-09-30: see [Kernel readiness corrections](KernelReadiness-v1.md#corrections-2026-09-30).] + +## Corrections (2026-09-30) + +**Evidence rule from 2026-09-30.** A row may read PASS only if it names a run id or check run on +the SHA in this sheet's header. Any other row reads NOT RUN, or CARRIED FORWARD with both SHAs. A +readiness PASS must cite a `Readiness summary` run whose verdict is `verified`. The full rule and +the four labels are in the [production-readiness README](README.md#evidence-rule-for-readiness-records-added-2026-09-30). + +**Which SHA.** This sheet names none. It was committed in `0ecb5bfc` (#110, 2026-05-22 01:17 UTC). +Its tree, `f1e22dfd`, is identical to PR #110's head `ee8befa7`. The gate scripts it cites did not exist in +any commit on 2026-05-19; [Kernel readiness](KernelReadiness-v1.md#corrections-2026-09-30) gives +the dates. May CI logs have expired (HTTP 410), so counts are static counts of xUnit test methods +the filter selects in the net8.0 build at `0ecb5bfc`. + +| Row | Label | Evidence | +|-----|-------|----------| +| Tier A | SUPPORTED, carried forward `ee8befa7` → `0ecb5bfc` (identical tree `f1e22dfd`) | Application Gate run 26262666898 (pull_request, `ee8befa7`, 2026-05-22), step "Tier A" success. At `0ecb5bfc` the workflow ran Tiers B–D only on `workflow_dispatch`, and none of its 468 runs is a dispatch. | +| Tier B | UNSUPPORTED-AT-THE-TIME (test leg, in part) | The filter names ValidateCommandTests, MeshDirectorCommandUriTests, TrustCommandTests and WorkflowCommandTests. At `0ecb5bfc`, three of these (ValidateCommandTests, TrustCommandTests, WorkflowCommandTests) derive from `UnitTestBase` and declare no xUnit fact. The CLI test project runs such suites only through `UnitTestBridgeTests.Framework_unit_test_passes`, whose fully qualified name contains none of those class names. So only MeshDirectorCommandUriTests (2 methods) could run. Security Tier C shows the same mechanism in CI: run 31982500018 (2026-08-17) printed "No test matches the given testcase filter `FullyQualifiedName~TrustCommandTests`" and then PASS. The `doctor --json` leg fails the tier only when `APPLICATION_GATE_STRICT_DOCTOR=1`. No CI run of this tier from May was found. | +| Tier C | UNEVIDENCED then; LATER-DECAYED | At `0ecb5bfc` the filter selects 4 methods: ApiDevelopmentHostDiTests (1) and FrameworkVirtualProdDemosTests (3). No CI run of it was found: the Application Gate workflow ran it only on dispatch and was never dispatched. The net8.0 build this tier uses dropped Tests/VirtualProduction on 2026-05-23 (`9579c813`) and Tests/API on 2026-08-17 (`e6b5579f`). On 2026-09-30 the script ran at `876cc527` in the Linux devtest container. It printed "No test matches the given testcase filter `FullyQualifiedName~ApiDevelopmentHostDiTests\|FullyQualifiedName~FrameworkVirtualProdDemosTests`", then `application-gate-tier-c: PASS`, and exited 0. | +| Tier D | SUPPORTED, carried forward `ee8befa7` → `0ecb5bfc` (identical tree `f1e22dfd`) | Prod dry run (Compose) run 26262666791 (pull_request, `ee8befa7`, 2026-05-22), job "agent-server stack" success. It runs the same prod-dry-run script with `--agent-server`. The tier itself prints PASS without running anything when Docker is absent. | diff --git a/docs/production-readiness/CompositionMeshReadiness-v1.md b/docs/production-readiness/CompositionMeshReadiness-v1.md index 71680e5c8..2f2d20537 100644 --- a/docs/production-readiness/CompositionMeshReadiness-v1.md +++ b/docs/production-readiness/CompositionMeshReadiness-v1.md @@ -1,6 +1,9 @@ # Composition & mesh readiness v1 **Status: COMPOSITION & MESH GATE GREEN** (Tiers A–D, 2026-05-19) +[corrected 2026-09-30: all four tiers are SUPPORTED by CI, each carried forward from another +commit. Tiers A–C ran on 2026-05-22 on PR #110's head `ee8befa7`, whose tree is identical to this +sheet's commit `0ecb5bfc`. Tier D ran on `7457e6c5` on 2026-05-19. See [Corrections](#corrections-2026-09-30).] Track after [Application Readiness v1](ApplicationReadiness-v1.md). **Plan:** [Composition & mesh hardening plan v1](CompositionMeshHardeningPlan-v1.md) @@ -14,16 +17,16 @@ make composition-mesh-gate-full # A–D (~2–5 min in-process; +10–15 min D | Date | Gate | Result | Notes | |------|------|--------|-------| -| 2026-05-19 | A–D local | **PASS** | 44 pipeline + 6 CLI + 18 fleet tests; mesh-lab workers E2E | +| 2026-05-19 | A–D local | **PASS** [corrected 2026-09-30: the local run left no artifact. CI backs each tier; see the table below.] | 44 pipeline + 6 CLI + 18 fleet tests; mesh-lab workers E2E | ## Tier summary | Tier | Focus | Command | Status | |------|--------|---------|--------| -| A | Pipeline composition | `make composition-mesh-gate-tier-a` | **PASS** (44 tests) | -| B | CLI pipeline + mesh | `make composition-mesh-gate-tier-b` | **PASS** (6 tests) | -| C | Mesh fleet in-process | `make composition-mesh-gate-tier-c` | **PASS** (18 tests) | -| D | Docker mesh workers | `make composition-mesh-gate-tier-d` | **PASS** | +| A | Pipeline composition | `make composition-mesh-gate-tier-a` | **PASS** (44 tests) [corrected 2026-09-30: SUPPORTED, carried forward ee8befa7 → 0ecb5bfc (PR #110 head, identical tree f1e22dfd), by Composition Mesh Gate run 26262666838 (2026-05-22). The filter selects exactly 44 test methods at 0ecb5bfc.] | +| B | CLI pipeline + mesh | `make composition-mesh-gate-tier-b` | **PASS** (6 tests) [corrected 2026-09-30: SUPPORTED, carried forward ee8befa7 → 0ecb5bfc (identical tree f1e22dfd), by the same run's "Tier A–C (PR default)" step. The filter selects exactly 6 test cases at 0ecb5bfc.] | +| C | Mesh fleet in-process | `make composition-mesh-gate-tier-c` | **PASS** (18 tests) [corrected 2026-09-30: SUPPORTED, carried forward ee8befa7 → 0ecb5bfc (identical tree f1e22dfd), by the same step. The filter selects exactly 18 test cases at 0ecb5bfc.] | +| D | Docker mesh workers | `make composition-mesh-gate-tier-d` | **PASS** [corrected 2026-09-30: SUPPORTED, carried forward 7457e6c5 → 0ecb5bfc. Mesh virtual lab gate run 26129468841 (2026-05-19) ran the workers profile and mesh-lab-verify.sh.] | ## Next: ship readiness @@ -35,5 +38,28 @@ See [Ship readiness v1](ShipReadiness-v1.md). ## Sign-off -- [x] `composition-mesh-gate-full` green (2026-05-19) -- [x] Application + kernel gates green same week +- [x] `composition-mesh-gate-full` green (2026-05-19) [corrected 2026-09-30: backed by CI runs + 26262666838 and 26129468841, both carried forward from other commits; see + [Corrections](#corrections-2026-09-30).] +- [x] Application + kernel gates green same week [corrected 2026-09-30: not fully backed; see + [Application](ApplicationReadiness-v1.md#corrections-2026-09-30) and [Kernel](KernelReadiness-v1.md#corrections-2026-09-30) corrections.] + +## Corrections (2026-09-30) + +**Evidence rule from 2026-09-30.** A row may read PASS only if it names a run id or check run on +the SHA in this sheet's header. Any other row reads NOT RUN, or CARRIED FORWARD with both SHAs. A +readiness PASS must cite a `Readiness summary` run whose verdict is `verified`. The full rule and +the four labels are in the [production-readiness README](README.md#evidence-rule-for-readiness-records-added-2026-09-30). + +**Which SHA.** This sheet names none. It was committed in `0ecb5bfc` (#110, 2026-05-22 01:17 UTC). +Its tree, `f1e22dfd`, is identical to PR #110's head `ee8befa7`. No commit contained these gate scripts on +2026-05-19. May CI logs have expired (HTTP 410), so only run and step conclusions survive. Counts +here are static counts of the xUnit test cases the filter selects in the net8.0 build at `0ecb5bfc`, +counting each theory row. All three match the counts this sheet recorded. + +| Row | Label | Evidence | +|-----|-------|----------| +| Tier A | SUPPORTED, carried forward `ee8befa7` → `0ecb5bfc` (identical tree `f1e22dfd`) | Composition Mesh Gate run 26262666838 (pull_request, `ee8befa7`, 2026-05-22), steps "Tier A (pipeline composition)" and "Tier A–C (PR default)" success. The filter `FullyQualifiedName~…Tests.Pipelines` selects 44 test methods at `0ecb5bfc`. None is a theory, so that is 44 test cases, matching the sheet's 44. | +| Tier B | SUPPORTED, carried forward `ee8befa7` → `0ecb5bfc` (identical tree `f1e22dfd`) | The same run's "Tier A–C (PR default)" step runs `make composition-mesh-gate`, which covers Tiers A, B and C. At `0ecb5bfc` the filter selects 6 test cases, matching the sheet's 6. MeshDirectorCommandUriTests contributes 3: a two-row theory and one fact. `UnitTestBridgeTests` contributes 3 rows, one each for MeshCommandTests, OptimizeAgentClusterScriptTests and PipelineCommandTests. These are the only `UnitTestBase` suites whose names match the DisplayName clause. | +| Tier C | SUPPORTED, carried forward `ee8befa7` → `0ecb5bfc` (identical tree `f1e22dfd`) | The same step. The filter selects 16 test methods at `0ecb5bfc`. One is a theory with 3 inline rows (MeshLabWorkerExecutorClientTests), so the total is 18 test cases, matching the sheet's 18. | +| Tier D | SUPPORTED, carried forward `7457e6c5` → `0ecb5bfc` | Mesh virtual lab gate run 26129468841 (push, `7457e6c5`, 2026-05-19) ran `docker compose --profile workers … up` and then passed mesh-lab-verify.sh, the same steps as `MESH_LAB_E2E_WORKERS=1` run-mesh-lab-e2e.sh. The compose file and scripts are byte-identical at `0ecb5bfc`. Without Docker, this tier prints a skip line and exits 0 without printing PASS, so a local "PASS" alone would not show that it ran. | diff --git a/docs/production-readiness/KernelReadiness-v1.md b/docs/production-readiness/KernelReadiness-v1.md index 460b3a3fa..e35fcff58 100644 --- a/docs/production-readiness/KernelReadiness-v1.md +++ b/docs/production-readiness/KernelReadiness-v1.md @@ -1,6 +1,9 @@ # Kernel Readiness v1 **Status: KERNEL READY FOR APPLICATIONS** (Tiers A–E automated gates green, 2026-05-19) +[corrected 2026-09-30: this sheet names no SHA, and no commit held these gates on 2026-05-19. CI +evidence exists for Tier A, for the mesh lab, and for parts of B–E. The Kernel Gate workflow never +ran Tiers B–E. See [Corrections](#corrections-2026-09-30).] **Plan:** [Kernel Hardening Plan v1](KernelHardeningPlan-v1.md) · **Quarterly ops:** [Kernel Chaos Drill v1](KernelChaosDrill-v1.md) @@ -14,18 +17,18 @@ make kernel-gate-full # A + B + C + D + E (~10–15 min first run; Docker for | Date | Gate | Result | |------|------|--------| -| 2026-05-19 | A–E local | **PASS** | -| 2026-05-19 | mesh-lab-e2e | **PASS** | +| 2026-05-19 | A–E local | **PASS** [corrected 2026-09-30: no single run backs this. Evidence per tier is in the table below.] | +| 2026-05-19 | mesh-lab-e2e | **PASS** [corrected 2026-09-30: SUPPORTED, carried forward 7457e6c5 → 0ecb5bfc. Mesh virtual lab gate run 26129468841 (2026-05-19) started the lab and passed mesh-lab-verify.sh and mesh-lab-verify-deep.sh.] | ## Tier summary | Tier | Focus | Command | Status | |------|--------|---------|--------| -| A | DI contracts, profiles | `make kernel-gate` | **PASS** | -| B | CLI + LiteDB resume | `make kernel-gate-tier-b` | **PASS** | -| C | ProdStyle, transport, air-gapped | `make kernel-gate-tier-c` | **PASS** | -| D | NuGet consumer sample | `make kernel-gate-tier-d` | **PASS** | -| E | OTel, perf, Compose dry run | `make kernel-gate-tier-e` | **PASS** | +| A | DI contracts, profiles | `make kernel-gate` | **PASS** [corrected 2026-09-30: SUPPORTED on 0ecb5bfc by Kernel Gate run 26262716833 (2026-05-22, step Tier A). Not on 2026-05-19: one of its three suites was first committed in #110.] | +| B | CLI + LiteDB resume | `make kernel-gate-tier-b` | **PASS** [corrected 2026-09-30: SUPPORTED, carried forward 7457e6c5 → 0ecb5bfc. The Kernel Gate workflow never ran this tier, but Production Readiness Gate v1 run 26129468793 (2026-05-19) ran the same steps or a superset of them on three OSes.] | +| C | ProdStyle, transport, air-gapped | `make kernel-gate-tier-c` | **PASS** [corrected 2026-09-30: partly SUPPORTED, carried forward 7457e6c5 → 0ecb5bfc. The gRPC ProdStyle leg passed as gRPC transport gate run 26129468823 (7457e6c5, 2026-05-19). The ProdStyle, workflow-executor and air-gapped legs are UNEVIDENCED.] | +| D | NuGet consumer sample | `make kernel-gate-tier-d` | **PASS** [corrected 2026-09-30: partly SUPPORTED. The runtime-graph build is covered by run 26262716833 on 0ecb5bfc. The pack-list check is carried forward 7457e6c5 → 0ecb5bfc from run 26129468816. The consumer sample is UNEVIDENCED.] | +| E | OTel, perf, Compose dry run | `make kernel-gate-tier-e` | **PASS** [corrected 2026-09-30: partly SUPPORTED, carried forward ee8befa7 → 0ecb5bfc (PR #110 head, identical tree f1e22dfd). The portal dry run passed in Prod dry run (Compose) run 26262666791. The OTel test and 3 perf tests are UNEVIDENCED.] | ## Tier E detail @@ -55,6 +58,41 @@ You may build on `application/src/` when `make kernel-gate-full` is green. Re-ru ## Sign-off -- [x] Tiers A–E automated (2026-05-19) +- [x] Tiers A–E automated (2026-05-19) [corrected 2026-09-30: the Kernel Gate workflow ran Tier A + only, from 2026-05-22 on. For B–E, see the tier table and [Corrections](#corrections-2026-09-30).] - [ ] Quarterly chaos drill checklist completed - [ ] Post-publish NuGet verify on release tag + +## Corrections (2026-09-30) + +**Evidence rule from 2026-09-30.** A row may read PASS only if it names a run id or check run on +the SHA in this sheet's header. Any other row reads NOT RUN, or CARRIED FORWARD with both SHAs. A +readiness PASS must cite a `Readiness summary` run whose verdict is `verified`. The full rule and +the four labels are in the [production-readiness README](README.md#evidence-rule-for-readiness-records-added-2026-09-30). + +**Which SHA.** This sheet names none. Its rows were committed in `0ecb5bfc` (#110) at +2026-05-22 01:17 UTC. Their tree is `f1e22dfd`, the same tree as PR #110's head `ee8befa7`. The +gate scripts these rows cite first appear in `43d5fca9`, PR #110's first commit, authored +2026-05-22 00:45 UTC. None of the cited make targets exists in the Makefile at `7457e6c5`, master +on 2026-05-19. A 2026-05-19 run was therefore of an uncommitted tree, and no SHA identifies it. +Outside docs, gate scripts, workflows and the Makefile, `7457e6c5` and `0ecb5bfc` differ only in +global.json, two helper scripts and the new test file KernelPhaseResolutionTests.cs. That is the +basis for carrying runs forward from `7457e6c5`. + +**Why test counts are missing.** GitHub keeps logs for 90 days. Every May log now returns HTTP 410, +so only the run and step conclusions below survive. Where a count is given, it is a static count of +xUnit test methods (not theory rows) that the tier's filter selects in the net8.0 build at +`0ecb5bfc`. That check was calibrated against security-gate run 31982500018 (2026-08-17) and +matched the zero or non-zero result of all four legs in that run. + +| Row | Label | Evidence | +|-----|-------|----------| +| Tier A | SUPPORTED on `0ecb5bfc` | Kernel Gate run 26262716833 (push, `0ecb5bfc`) and run 26262666837 (PR head `ee8befa7`, same tree), 2026-05-22, step "Tier A" success. The filters select 22 + 14 test methods. KernelPhaseResolutionTests was added in #110, so this tier as written could not have run on a committed tree on 2026-05-19. | +| Tier B | SUPPORTED, carried forward `7457e6c5` → `0ecb5bfc` | Kernel Gate's Tier B step runs only on `workflow_dispatch`, and none of its 908 runs is a dispatch. Production Readiness Gate v1 run 26129468793 (push, `7457e6c5`, 2026-05-19, ubuntu, macOS and windows jobs all success) ran the same three builds, the pipeline tests, the same CLI validate/run/fallback and the same LiteDB cross-process resume. Its filter `FullyQualifiedName~Pipelines` is a superset of this tier's. | +| Tier C | Partly SUPPORTED, carried forward `7457e6c5` → `0ecb5bfc`; rest UNEVIDENCED | The gRPC transport gate run 26129468823 (`7457e6c5`, 2026-05-19) ran this tier's `Category=ProdStyle` transport leg. The ProdStyle infrastructure leg (68 methods), workflow executor (12) and air-gapped (5) legs have no CI run that was found. | +| Tier D | Partly SUPPORTED (build on `0ecb5bfc`; pack-list check carried forward `7457e6c5` → `0ecb5bfc`); rest UNEVIDENCED | Run 26262716833 above builds the runtime solution. Pack hosting graph alignment run 26129468816 (`7457e6c5`, 2026-05-19) ran the pack-list check. No CI run of the StableSdkHostSample consumer was found. | +| Tier E | Partly SUPPORTED, carried forward `ee8befa7` → `0ecb5bfc` (identical tree `f1e22dfd`); rest UNEVIDENCED | Prod dry run (Compose) run 26262666791 (PR head `ee8befa7`, 2026-05-22) passed its "portal stack" job, which runs the same prod-dry-run script. The OpenTelemetry test (1 method) and the orchestration performance tests (3 methods, which matches "3" in Tier E detail) have no CI run that was found. | +| mesh-lab-e2e | SUPPORTED, carried forward `7457e6c5` → `0ecb5bfc` | Mesh virtual lab gate run 26129468841 (push, `7457e6c5`, 2026-05-19 22:35 UTC) started the lab with the workers profile, then passed mesh-lab-verify.sh and mesh-lab-verify-deep.sh. docker-compose.mesh-lab.yml, run-mesh-lab-e2e.sh and both verify scripts are byte-identical at `0ecb5bfc`. | + +These notes record only what was true for this sheet in May. They say nothing about the kernel +today. To re-sign the sheet, run `make kernel-gate-full` on a named SHA and record a run id there. diff --git a/docs/production-readiness/OpsReadiness-v1.md b/docs/production-readiness/OpsReadiness-v1.md index 806090fad..ceb0d301f 100644 --- a/docs/production-readiness/OpsReadiness-v1.md +++ b/docs/production-readiness/OpsReadiness-v1.md @@ -1,6 +1,8 @@ # Operations & dogfood readiness v1 **Status: OPS GATE GREEN** (Tiers A–C, E default profile, 2026-05-19) +[corrected 2026-09-30: UNEVIDENCED. The Ops Gate workflow could only be dispatched by hand. Its +first run was 2026-08-16 and it failed. See [Corrections](#corrections-2026-09-30).] Track after [Ship readiness v1](ShipReadiness-v1.md). **Plan:** [Ops hardening plan v1](OpsHardeningPlan-v1.md) @@ -15,17 +17,17 @@ make ashlar-ready-gate # full stack (use ASHLAR_READY_SKIP_DOCKER=1 locally fo | Date | Gate | Result | Notes | |------|------|--------|-------| -| 2026-05-19 | A–C, E local | **PASS** | Tier D mesh/chaos optional | +| 2026-05-19 | A–C, E local | **PASS** [corrected 2026-09-30: UNEVIDENCED. The Ops Gate workflow first ran on 2026-08-16.] | Tier D mesh/chaos optional | ## Tier summary | Tier | Focus | Status | |------|--------|--------| -| A | Dogfood blocks 1–6 | **PASS** | -| B | Dogfood blocks 7–9 + IPC mesh | **PASS** | -| C | Closed-loop self-improvement | **PASS** | +| A | Dogfood blocks 1–6 | **PASS** [corrected 2026-09-30: UNEVIDENCED. Each block's filter selects one test method; no CI run found.] | +| B | Dogfood blocks 7–9 + IPC mesh | **PASS** [corrected 2026-09-30: UNEVIDENCED. Each filter selects one test method; no CI run found.] | +| C | Closed-loop self-improvement | **PASS** [corrected 2026-09-30: UNEVIDENCED. The filter selects one test method; no CI run found.] | | D | Mesh deep / chaos-lite | optional | -| E | Oh-shit demo (quick) | **PASS** | +| E | Oh-shit demo (quick) | **PASS** [corrected 2026-09-30: UNEVIDENCED. No CI run found.] | ## Next: security & trust @@ -37,5 +39,25 @@ See [Security readiness v1](SecurityReadiness-v1.md). ## Sign-off -- [x] `ops-gate` default tiers green (2026-05-19) -- [x] Ship gate green same week +- [x] `ops-gate` default tiers green (2026-05-19) [corrected 2026-09-30: UNEVIDENCED; see + [Corrections](#corrections-2026-09-30).] +- [x] Ship gate green same week [corrected 2026-09-30: only Tier A is backed; see + [Ship corrections](ShipReadiness-v1.md#corrections-2026-09-30).] + +## Corrections (2026-09-30) + +**Evidence rule from 2026-09-30.** A row may read PASS only if it names a run id or check run on +the SHA in this sheet's header. Any other row reads NOT RUN, or CARRIED FORWARD with both SHAs. A +readiness PASS must cite a `Readiness summary` run whose verdict is `verified`. The full rule and +the four labels are in the [production-readiness README](README.md#evidence-rule-for-readiness-records-added-2026-09-30). + +**Which SHA.** This sheet names none. It was committed in `0ecb5bfc` (#110, 2026-05-22 01:17 UTC). +No commit contained the ops-gate scripts on 2026-05-19. At `0ecb5bfc`, +`.github/workflows/ops-gate.yml` triggers only on `workflow_dispatch`. Its first run was 31972469016 +on 2026-08-16, which failed. Of its 97 runs, 3 succeeded. The workflow has no May run of any tier. + +**Could the tiers have run tests then?** Yes. At `0ecb5bfc` each dogfood filter (DogfoodBlock1Tests +through DogfoodBlock9Tests, DogfoodBlock9LocalIpcTests, DogfoodClosedLoopTests) selects exactly one +xUnit test method in the net8.0 build. These rows are therefore not disproven. They are UNEVIDENCED: +the local run left no artifact, and no other run found covers these tiers. Under the rule above, +they read NOT RUN. Tier E (`oh-shit-demo.sh --quick --no-build`) has no CI run that was found either. diff --git a/docs/production-readiness/RCReadiness-v1.md b/docs/production-readiness/RCReadiness-v1.md index 188fda804..d74ab58cb 100644 --- a/docs/production-readiness/RCReadiness-v1.md +++ b/docs/production-readiness/RCReadiness-v1.md @@ -16,7 +16,10 @@ make kernel-coverage-gate ## Sign-off -- [x] `make waterproofing-gate-full` green (local, SHA `bec2a6ed`) +- [x] `make waterproofing-gate-full` green (local, SHA `bec2a6ed`) [corrected 2026-09-30: + UNEVIDENCED as a whole. `.github/workflows/waterproofing-gate.yml` only runs when dispatched + by hand and has never run. Three of its four stages passed CI on `bec2a6ed`, but only after + this sign-off was committed. See [Release sign-off corrections](ReleaseSignOff-v1.md#corrections-2026-09-30).] - [x] [Release sign-off v1](ReleaseSignOff-v1.md) and [Rollback drill v1](RollbackDrill-v1.md) recorded - [ ] `rc-gate-tier-d` strict on `master` (`RC_GATE_TRIGGER_GH=1 make rc-gate-tier-d`) - [ ] GitHub RC workflows green on `master` (see `.ashlar/rc-gate/github-workflows.txt`) @@ -28,3 +31,12 @@ make waterproofing-gate-full ``` See [Perf readiness](PerfReadiness-v1.md), [Compat readiness](CompatReadiness-v1.md), [DR readiness](DRReadiness-v1.md). + +## Evidence rule (added 2026-09-30) + +A sign-off line may be ticked as green only if it names a run id or check run on the SHA it +signs. Any other line reads NOT RUN, or CARRIED FORWARD with both SHAs. A readiness PASS must cite +a `Readiness summary` run whose verdict is `verified`. The full rule is in the +[production-readiness README](README.md#evidence-rule-for-readiness-records-added-2026-09-30). +The waterproofing line above is corrected in place. The evidence is set out in +[Release sign-off corrections](ReleaseSignOff-v1.md#corrections-2026-09-30). diff --git a/docs/production-readiness/README.md b/docs/production-readiness/README.md index 376a7192e..f39b88e81 100644 --- a/docs/production-readiness/README.md +++ b/docs/production-readiness/README.md @@ -27,6 +27,49 @@ This folder helps you **support every major production concern** in one place: r | Operators | [Operator deployment](OperatorDeployment.md) | Production install, upgrades, backups pointer | | Audience fit | [Catalog by deployment type](CatalogByDeploymentType.md) | SMB vs enterprise vs SaaS: which items matter most | +## Evidence rule for readiness records (added 2026-09-30) + +> **Read this before trusting any PASS in this folder.** An audit on 2026-09-30 re-checked every +> PASS in the `*Readiness-v1.md` sheets and in [Release sign-off v1](ReleaseSignOff-v1.md) against +> what was true on each sheet's own date and SHA. Some rows held up. Others rested on a local run +> that left no artifact, on a tier no CI workflow had run by then, or on a `dotnet test` step whose +> filter selected no tests. `dotnet test` exits 0 when nothing matches, so a green step does not +> show that a test ran. Those rows now carry a dated `[corrected 2026-09-30: …]` note in place. +> Nothing was deleted or re-ticked. + +From 2026-09-30 on, these rules apply: + +- A row may read **PASS** only if it names a GitHub Actions run id or check run **on the SHA in + the sheet's header**. For a `dotnet test` leg, the row also gives the executed count from that + run's log. +- Any other row reads **NOT RUN**, or **CARRIED FORWARD** with both SHAs: the one the cited run + tested, and the one the sheet is for. This includes a run on a different commit with the same + tree, such as a pull request head and the commit it merged as. +- A readiness PASS must cite a `Readiness summary` run whose verdict is `verified`. A verdict of + `partial`, `not-verified` or `failed` does not count. #685 added these verdicts; each run shows + its verdict in the step summary and job outputs. +- A local run counts as evidence only if its output is committed or attached beside the row. + `.ashlar/` (`.nexo/` before the rename) is gitignored, so a report left there does not count. +- Nothing enforces this rule yet. No test or gate reads these tables. `scripts/rc-gate-tier-e.sh`, + which `make waterproofing-gate-full` runs, reports "sign-off: checked items found" for any + ticked line naming Product or Engineering. It still says that about + [Release sign-off v1](ReleaseSignOff-v1.md), whose Engineering tick is now annotated + UNSUPPORTED-AT-THE-TIME. Apply the rule by review until a check reads it. + +The corrections use four labels: + +- **SUPPORTED** names the run. If the run tested a different commit from the sheet's SHA, the + note says "carried forward", gives both SHAs as tested → sheet, and says why the run still + covers the row. If the two commits have the same tree, the note names the tree hash. If the + trees differ, the note says what differs, or that the files the row depends on are + byte-identical. A run that finished after the sheet was signed says "after sign-off". +- **UNSUPPORTED-AT-THE-TIME** means the cited command could not have shown what the row claims, + even at the sheet's own SHA. +- **LATER-DECAYED** means the row was true then and is not now. The note gives the date and cause. +- **UNEVIDENCED** means a local run that nobody can inspect now. The tier's own workflow did not + run it, and no run found on the sheet's SHA or its parent ran the same commands. It is not + disproven, but under this rule the row reads NOT RUN. + ## How to use this 1. Read [Catalog by deployment type](CatalogByDeploymentType.md) and pick your lane. diff --git a/docs/production-readiness/ReleaseSignOff-v1.md b/docs/production-readiness/ReleaseSignOff-v1.md index 4bd9ddd58..cf96f4389 100644 --- a/docs/production-readiness/ReleaseSignOff-v1.md +++ b/docs/production-readiness/ReleaseSignOff-v1.md @@ -20,24 +20,62 @@ Complete before tagging a release candidate. - [x] Product: scope and known limitations accepted (RC gate stack; strict GH workflows pending green on `master`) - [x] Engineering: all automated gates green for target SHA (local: `waterproofing-gate-full`, `ship-gate-tier-d` with runtime gate) + [corrected 2026-09-30: UNSUPPORTED-AT-THE-TIME. 8 of the 19 workflow runs on `bec2a6ed` + concluded failure, including the Full Platform Readiness Gate and its `Readiness summary` + job. When this box was ticked, 21 minutes after `bec2a6ed`, no run on that SHA had succeeded + and four had already shown a failure. See [Corrections](#corrections-2026-09-30).] - [x] Security: exceptions file reviewed (`docs/exceptions.yaml` — no High/Critical entries) + [corrected 2026-09-30: SUPPORTED. `git show bec2a6ed:docs/exceptions.yaml` reads + `exceptions: []`.] - [x] Operations: rollback drill recorded ([Rollback drill v1](RollbackDrill-v1.md)) + [corrected 2026-09-30: the record exists; `fb544764` filled it in with this tick. The checks + it lists passed CI on `bec2a6ed` only after sign-off (see the DR row below). It records a + restore in place, not a rollback to an earlier version: "To version" reads N/A. Its local run + left nothing to inspect. The reports went to gitignored `.nexo/`, and the mesh task id it + cites does not date the run. See [Corrections](#corrections-2026-09-30).] ## Sign-off record | Role | Name | Date | SHA | |------|------|------|-----| | Product | Ian Frelinger | 2026-05-22 | `bec2a6ed` | -| Engineering | Ashlar readiness gates (automated) | 2026-05-22 | `bec2a6ed` | -| Security | Ashlar security-gate (automated) | 2026-05-22 | `bec2a6ed` | -| Operations | DR gate + mesh-lab persistence | 2026-05-22 | `bec2a6ed` | +| Engineering | Ashlar readiness gates (automated) [corrected 2026-09-30: see the Engineering item above; CI on this SHA did not come out green] | 2026-05-22 | `bec2a6ed` | +| Security | Ashlar security-gate (automated) [corrected 2026-09-30: SUPPORTED for Tiers A–B only, carried forward `2a1b4d1a` → `bec2a6ed` (PR #114 head, identical tree `b2996d47`), run 26262735412, after sign-off; Tiers C–E not run] | 2026-05-22 | `bec2a6ed` | +| Operations | DR gate + mesh-lab persistence [corrected 2026-09-30: DR Tiers A–B SUPPORTED by dr-gate run 26262738375; mesh director persistence SUPPORTED by Mesh virtual lab gate run 26262738372; both on this SHA, after sign-off] | 2026-05-22 | `bec2a6ed` | ## Local gate evidence (2026-05-22) | Gate | Command | Result | |------|---------|--------| -| Waterproofing | `make waterproofing-gate-full` | PASS | -| Ship runtime | `SHIP_GATE_RUN_RUNTIME_GATE=1 make ship-gate-tier-d` | PASS | -| DR | `make dr-gate-full` | PASS (mesh director persistence verified) | +| Waterproofing | `make waterproofing-gate-full` | PASS [corrected 2026-09-30: UNEVIDENCED as a whole. The workflow has never run. Perf, compat and DR later passed CI on this SHA. RC policy has no CI run on this SHA. One compat step has selected zero tests since 2026-06-08.] | +| Ship runtime | `SHIP_GATE_RUN_RUNTIME_GATE=1 make ship-gate-tier-d` | PASS [corrected 2026-09-30: UNEVIDENCED. Ship Gate first ran on 2026-08-16.] | +| DR | `make dr-gate-full` | PASS (mesh director persistence verified) [corrected 2026-09-30: Tiers A–B SUPPORTED by run 26262738375, after sign-off. In that run, Tier C took its placeholder path. Mesh director persistence is SUPPORTED after sign-off by Mesh virtual lab gate run 26262738372 on this SHA, which ran the restart check that Tier C calls. The mesh task id in the rollback drill record does not date this local run. It dates the first creation of a task that the restart check reuses on every run against the same lab volume, so it cannot show which tree this run tested; see the Operations row.] | GitHub RC workflows: triggered on `master` after merge #114; run `RC_GATE_TRIGGER_GH=1 make rc-gate-tier-d` for strict verification. + +## Corrections (2026-09-30) + +**Evidence rule from 2026-09-30.** A row may read PASS only if it names a run id or check run on +the SHA in this sheet's header (`bec2a6ed`). Any other row reads NOT RUN, or CARRIED FORWARD with +both SHAs. A readiness PASS must cite a `Readiness summary` run whose verdict is `verified`. The +full rule and the four labels are in the +[production-readiness README](README.md#evidence-rule-for-readiness-records-added-2026-09-30). +Add a new row for a new release; do not re-tick these. + +**Timeline.** `bec2a6ed` (#114) was committed at 2026-05-22 01:17:43 UTC. This sign-off was +committed in `fb544764` (#115) at 01:38:36 UTC; PR #115 was opened and merged within two seconds. +May logs have expired (HTTP 410). Run and job conclusions and times survive. + +| Item | Label | Evidence | +|------|-------|----------| +| Engineering: all automated gates green for target SHA | UNSUPPORTED-AT-THE-TIME | GitHub lists 19 workflow runs with head SHA `bec2a6ed`. 8 concluded failure: Full Platform Readiness Gate 26262738366, whose `Readiness summary` job also failed; Cross-Platform Tests 26262738322, with Build & Test failing on ubuntu, macOS and windows; Persistence Tests (Multi-OS) 26262738364; Runtime Release Gate 26262738371; Test Caching Multi-Environment 26262738380; Optimize Agent Cluster Gate 26262738370; and runs 26262737929 and 26262737725 of onboarding-quickstart-gate.yml and mapbox-tile-helpers-ci.yml. At 01:38:36 UTC none of the 19 had succeeded (the first success completed at 01:45:21). Four already showed a failure. mapbox-tile-helpers-ci.yml run 26262737725 at 01:17:45 and onboarding-quickstart-gate.yml run 26262737929 at 01:17:46 were each recorded as failed in the second they were created. Cross-Platform Tests 26262738322 had a failed windows job at 01:36:16; the run itself finished at 01:37:36, after its ubuntu job failed at 01:37:35. Persistence Tests 26262738364 had a failed macOS job at 01:36:26. The jobs API lists five Caching Tests jobs under 26262737725, all completed after 02:11, but that run's record already read failure at 01:17:45. | +| Waterproofing, `make waterproofing-gate-full` | UNEVIDENCED as a whole | `.github/workflows/waterproofing-gate.yml` triggers only on `workflow_dispatch`, and `gh run list --workflow=waterproofing-gate.yml` returns no runs (API `total_count` 0). The script runs perf, compat, DR and then rc-gate-tier-e. Three stages ran in CI on this SHA with the same scripts and success: perf-gate 26262738323 (done 01:49:07), dr-gate 26262738375 (01:50:21) and compat-gate 26262738367 (01:56:14). All three finished after the sign-off. The RC-policy stage (rc-gate-tier-e) has no CI run on this SHA: at `bec2a6ed`, rc-gate.yml is dispatch-only and runs Tiers A–D, and its first run was on 2026-05-28. | +| Compat, inside waterproofing | SUPPORTED after sign-off, by compat-gate run 26262738367 on `bec2a6ed`; LATER-DECAYED (one step) | compat-gate run 26262738367 (push, `bec2a6ed`) ran `COMPAT_GATE_SKIP_PRIOR=1 bash scripts/compat-gate.sh`, the line waterproofing-gate.sh runs, and succeeded at 01:56:14, 18 minutes after the sign-off. At `bec2a6ed`, compat Tier A's first step, filter `FullyQualifiedName~MeshTaskExecutionServiceTests.MigrateForCheckpointAsync`, selects 1 test method. `678d0718` (#151, 2026-06-08) moved that test into the commercial test tree, but the script still runs the filter against the infrastructure test project. CI runs 32612595835 (2026-08-23) and 34687917953 (2026-09-12), and a Linux devtest container run at `876cc527` on 2026-09-30, all print "No test matches the given testcase filter" for that step and then `compat-gate-tier-a: PASS`. The other two Tier A steps still ran 1 and 4 tests in each. | +| Ship runtime, `SHIP_GATE_RUN_RUNTIME_GATE=1 make ship-gate-tier-d` | UNEVIDENCED | ship-gate.yml was dispatch-only and first ran on 2026-08-16 (run 31972468094). No run exists on this SHA. | +| DR, `make dr-gate-full` | Tiers A–B SUPPORTED after sign-off; "(mesh director persistence verified)" SUPPORTED after sign-off, by the mesh-lab gate | dr-gate run 26262738375 on `bec2a6ed` ran `DR_GATE_SKIP_PRIOR=1 bash scripts/dr-gate.sh`. That is the same command, because 1 is the default. It succeeded 12 minutes after the sign-off. dr-gate-tier-c.sh checks mesh-director persistence only when .env.mesh-lab exists. That file is not in the tree and dr-gate.yml does not write it, so the CI run took the advisory path: a copy of a placeholder file, reported as "PASS (advisory)". The persistence claim is supported by a different workflow on the same SHA. At `bec2a6ed`, mesh-lab-verify.sh calls mesh-lab-verify-persistence.sh, the script dr-gate-tier-c.sh calls on its full path, unless MESH_LAB_SKIP_PERSISTENCE_VERIFY is set. mesh-lab-gate.yml does not set it. Its "Write lab env" step writes an API key and sets the director persistence provider to LiteDb. The persistence script skips only when the key is missing or the provider is not LiteDb. Otherwise it restarts peer-a and exits 1 if the seeded fleet node or task is gone. Mesh virtual lab gate run 26262738372 (push, `bec2a6ed`) passed its "Run verify scripts" step and succeeded at 02:23:46, 45 minutes after the sign-off. Its log has expired. Run 29668620853 on `d24b2ba9` (2026-07-19), whose persistence script is byte-identical, shows the same step printing "Mesh task survived restart". | +| Security sign-off (security-gate, automated) | SUPPORTED for Tiers A–B, carried forward `2a1b4d1a` → `bec2a6ed`, after sign-off | security-gate does not run on push, so there is no run on `bec2a6ed` itself. Run 26262735412 ran on PR #114's head `2a1b4d1a`, whose tree `b2996d47` is identical to `bec2a6ed`'s. Tiers A and B succeeded, finishing at 01:57:20 and 01:57:29, after the sign-off. The workflow did not run Tiers C–E. | +| Security: exceptions file reviewed | SUPPORTED | `git show bec2a6ed:docs/exceptions.yaml` shows `exceptions: []`. | +| Operations: rollback drill recorded | Record present; its checks SUPPORTED after sign-off; no rollback to an earlier version done | At `bec2a6ed`, [Rollback drill v1](RollbackDrill-v1.md) read "_not yet recorded_". `fb544764`, the commit that ticked this box, filled it in: operator `make dr-gate-full` on a local mesh lab, "From version `bec2a6ed`", "Result PASS". The checks it lists (DR Tiers A–B and the mesh director restart) passed CI on `bec2a6ed` only after sign-off; see the DR row. The local run itself cannot be inspected. The record cites reports under `.nexo/dr-gate/`, `.nexo/` is in `.gitignore` at `bec2a6ed`, and `fb544764` tracks no file under it. Its RPO and RTO figures come only from that run. It also reads "To version: N/A (restore-in-place drill, not version downgrade)", so step 2 of its own procedure, deploying the previous known-good artifact, was not done. The mesh task id it cites, `20260522001559314-7ab02c30…`, does not date the drill. At `bec2a6ed` the id is `DateTimeOffset.UtcNow:yyyyMMddHHmmssfff` plus a GUID (src/Nexo.Infrastructure/Fleet/LiteDbMeshTaskRegistry.cs:22), so peer-a created that task at 00:15:59 UTC by its own clock. That is before `bec2a6ed` (01:17:43) and before any DR gate script was committed (`edb20938`, authored 00:45:13). But mesh-lab-verify-persistence.sh always sends the idempotency key `mesh-lab-persist-idem`. `POST /api/mesh/tasks` returns the existing task for a known key (application/src/Nexo.API/Endpoints/NexoEndpoints.cs:935–940). peer-a keeps tasks in LiteDB on the named volume `mesh_lab_peer_a_data`, which neither `make mesh-lab-up` nor dr-gate-tier-c.sh removes, and the task registry has no delete. So every later run against that volume prints the same id. The id dates the first seeding of the task, not this drill, and cannot show which tree the drill ran on. | + +These notes do not re-sign anything. For a release, add a row that names its SHA and the runs on +that SHA, including a `Readiness summary` run with verdict `verified`. diff --git a/docs/production-readiness/SecurityReadiness-v1.md b/docs/production-readiness/SecurityReadiness-v1.md index 6bcd98e02..72aeb0fdf 100644 --- a/docs/production-readiness/SecurityReadiness-v1.md +++ b/docs/production-readiness/SecurityReadiness-v1.md @@ -1,6 +1,10 @@ # Security & trust readiness v1 **Status: SECURITY GATE GREEN** (Tiers A–E, 2026-05-19) +[corrected 2026-09-30: Tiers A and B are SUPPORTED, carried forward from CI on PR #110's head +`ee8befa7`, whose tree is identical to this sheet's commit `0ecb5bfc`. But Tier B has +selected zero tests since 2026-08-17. Tier C's unit-suite step selected zero tests even then. Tiers +D and E are UNEVIDENCED. See [Corrections](#corrections-2026-09-30).] Track after [Ops readiness v1](OpsReadiness-v1.md). **Plan:** [Security hardening plan v1](SecurityHardeningPlan-v1.md) @@ -14,17 +18,17 @@ make security-gate-full | Date | Gate | Result | Notes | |------|------|--------|-------| -| 2026-05-19 | A–E local | **PASS** | Supply-chain: app + Hosting + Infrastructure; xunit deprecated warning | +| 2026-05-19 | A–E local | **PASS** [corrected 2026-09-30: no single run backs this. Evidence per tier is in the table below.] | Supply-chain: app + Hosting + Infrastructure; xunit deprecated warning | ## Tier summary | Tier | Focus | Command | Status | |------|--------|---------|--------| -| A | Trust core | `make security-gate-tier-a` | **PASS** | -| B | API security middleware | `make security-gate-tier-b` | **PASS** | -| C | Trust CLI | `make security-gate-tier-c` | **PASS** | -| D | Supply chain | `make security-gate-tier-d` | **PASS** | -| E | Air-gapped + safety | `make security-gate-tier-e` | **PASS** | +| A | Trust core | `make security-gate-tier-a` | **PASS** [corrected 2026-09-30: SUPPORTED, carried forward ee8befa7 → 0ecb5bfc (PR #110 head, identical tree f1e22dfd), by Security Gate run 26262666843 (2026-05-22, step Tier A). 34 test methods were selectable.] | +| B | API security middleware | `make security-gate-tier-b` | **PASS** [corrected 2026-09-30: SUPPORTED then, carried forward ee8befa7 → 0ecb5bfc (identical tree f1e22dfd), by the same run (step Tier B; 24 methods selectable). LATER-DECAYED: zero tests since e6b5579f (2026-08-17). Run 36642632648 (2026-09-29) prints "No test matches" and then PASS.] | +| C | Trust CLI | `make security-gate-tier-c` | **PASS** [corrected 2026-09-30: UNSUPPORTED-AT-THE-TIME for its unit-suite step, whose filter could select no test. The trust boundary and dashboard JSON smokes are UNEVIDENCED.] | +| D | Supply chain | `make security-gate-tier-d` | **PASS** [corrected 2026-09-30: UNEVIDENCED. The Security Gate workflow first ran it on 2026-08-17, and no report survives; see the sign-off note below.] | +| E | Air-gapped + safety | `make security-gate-tier-e` | **PASS** [corrected 2026-09-30: UNEVIDENCED. 30 test methods were selectable, but the Security Gate workflow first ran it on 2026-08-17.] | ## Next: release candidate @@ -36,7 +40,8 @@ See [RC hardening plan v1](RCHardeningPlan-v1.md) and [Release candidate checkli ## Sign-off -- [x] `security-gate-full` green (2026-05-19) +- [x] `security-gate-full` green (2026-05-19) [corrected 2026-09-30: only Tiers A and B are + backed, and B has since decayed; see [Corrections](#corrections-2026-09-30).] - [x] Tier D reports in `.ashlar/security-gate/` (no High/Critical in scanned surfaces) — **the reports themselves are not reviewable from this repository.** `.ashlar/` is in `.gitignore` and nothing under it is tracked, so the "no High/Critical" half of this line rests on a @@ -44,4 +49,27 @@ See [RC hardening plan v1](RCHardeningPlan-v1.md) and [Release candidate checkli writes `vulnerable-packages.txt` and `deprecated-packages.txt` under that directory; re-run it and read them rather than taking this line's word for it. Re-signing this without a re-run would be signing for a scan nobody has seen. -- [x] Air-gapped tier verified (in-process safety + profile tests) +- [x] Air-gapped tier verified (in-process safety + profile tests) [corrected 2026-09-30: + UNEVIDENCED; see Tier E below.] + +## Corrections (2026-09-30) + +**Evidence rule from 2026-09-30.** A row may read PASS only if it names a run id or check run on +the SHA in this sheet's header. Any other row reads NOT RUN, or CARRIED FORWARD with both SHAs. A +readiness PASS must cite a `Readiness summary` run whose verdict is `verified`. The full rule and +the four labels are in the [production-readiness README](README.md#evidence-rule-for-readiness-records-added-2026-09-30). + +**Which SHA.** This sheet names none. It was committed in `0ecb5bfc` (#110, 2026-05-22 01:17 UTC). +Its tree, `f1e22dfd`, is identical to PR #110's head `ee8befa7`. No commit contained the security-gate scripts +on 2026-05-19. At `0ecb5bfc`, pull requests ran only Tiers A and B, and Tiers C–E ran only on +`workflow_dispatch`. The first dispatch was run 31982500018 on 2026-08-17. May CI logs have expired +(HTTP 410). Counts marked "selectable" are static counts of xUnit test methods the filter selects +in the net8.0 build at `0ecb5bfc`. Counts marked "passed" come from CI logs. + +| Row | Label | Evidence | +|-----|-------|----------| +| Tier A | SUPPORTED, carried forward `ee8befa7` → `0ecb5bfc` (identical tree `f1e22dfd`) | Security Gate run 26262666843 (pull_request, `ee8befa7`, 2026-05-22), step "Tier A (trust core)" success. 34 methods were selectable at `0ecb5bfc`. For comparison, the tier passed 97 tests in run 31982500018 (2026-08-17) and 97 in run 36642632648 (2026-09-29). | +| Tier B | SUPPORTED then, carried forward `ee8befa7` → `0ecb5bfc` (identical tree `f1e22dfd`); LATER-DECAYED | Same run, step "Tier B (API security)" success. At `0ecb5bfc` the filter selects 24 methods from the four API security suites under Tests/API. Run 31982500018 (2026-08-17) passed 39 tests. `e6b5579f` (committed 2026-08-17 22:27 -04:00) removed Tests/API from the net8.0 build, which is the build this tier tests. Since then, run 32608958552 (2026-08-23) and run 36642632648 (2026-09-29) both print "No test matches the given testcase filter" and then `security-gate-tier-b: PASS`. The fifth class it names, SecurityAnalysisRuleTests, is a `UnitTestBase` suite with no xUnit test. | +| Tier C | UNSUPPORTED-AT-THE-TIME (unit-suite step) | At `0ecb5bfc`, TrustCommandTests derives from `UnitTestBase` and declares no xUnit test. The CLI test project runs such suites only through `UnitTestBridgeTests.Framework_unit_test_passes`, whose fully qualified name does not contain "TrustCommandTests". The script was unchanged on 2026-08-17. Its first CI run, 31982500018, printed "No test matches the given testcase filter `FullyQualifiedName~TrustCommandTests`" and then `security-gate-tier-c: PASS`. The trust boundary and dashboard JSON smokes have no May run (UNEVIDENCED). The filter now selects through `UnitTestBridgeTests` with a DisplayName clause, and run 36642632648 passed 61 tests. | +| Tier D | UNEVIDENCED | The workflow has no May run of it. Its first CI execution was run 31982500018 on `5d0e2fd3` (2026-08-17), a different SHA, which printed `security-gate-tier-d: PASS`. The sign-off note above explains why the 2026-05-19 reports cannot be reviewed. | +| Tier E | UNEVIDENCED | 30 methods were selectable at `0ecb5bfc`, but the workflow has no May run of it. Its first CI execution, run 31982500018 (2026-08-17), passed 50 tests on a different SHA. | diff --git a/docs/production-readiness/ShipReadiness-v1.md b/docs/production-readiness/ShipReadiness-v1.md index d0cdbca0d..45431913d 100644 --- a/docs/production-readiness/ShipReadiness-v1.md +++ b/docs/production-readiness/ShipReadiness-v1.md @@ -1,6 +1,10 @@ # Ship readiness v1 **Status: SHIP GATE GREEN** (Tiers A–D default profile, 2026-05-19) +[corrected 2026-09-30: the Ship Gate workflow could only be dispatched by hand, and its first run +was 2026-08-16. Tier A is SUPPORTED, carried forward from an equivalent 2026-05-19 run on +`7457e6c5`. Tiers B–D are UNEVIDENCED. +See [Corrections](#corrections-2026-09-30).] Track after [Composition & mesh readiness v1](CompositionMeshReadiness-v1.md). **Plan:** [Ship hardening plan v1](ShipHardeningPlan-v1.md) @@ -14,16 +18,16 @@ make ship-gate-full | Date | Gate | Result | Notes | |------|------|--------|-------| -| 2026-05-19 | A–D local | **PASS** | Runtime bundle opt-in (`SHIP_GATE_RUN_RUNTIME_GATE=1`) | +| 2026-05-19 | A–D local | **PASS** [corrected 2026-09-30: only Tier A is backed, by an equivalent run. See the table below.] | Runtime bundle opt-in (`SHIP_GATE_RUN_RUNTIME_GATE=1`) | ## Tier summary | Tier | Focus | Status | |------|--------|--------| -| A | Production Readiness Gate v1 CLI + LiteDB resume | **PASS** | -| B | ProdStyle + smoke + doctor | **PASS** | -| C | Release preflight (`0.0.0-ship-gate-local`) | **PASS** | -| D | Doctor sign-off | **PASS** | +| A | Production Readiness Gate v1 CLI + LiteDB resume | **PASS** [corrected 2026-09-30: SUPPORTED, carried forward 7457e6c5 → 0ecb5bfc. Production Readiness Gate v1 run 26129468793 (2026-05-19, three OSes) ran the steps this tier mirrors.] | +| B | ProdStyle + smoke + doctor | **PASS** [corrected 2026-09-30: UNEVIDENCED. No CI run found; 68 + 9 test methods were selectable, and the doctor leg cannot fail the tier.] | +| C | Release preflight (`0.0.0-ship-gate-local`) | **PASS** [corrected 2026-09-30: UNEVIDENCED. No CI run found.] | +| D | Doctor sign-off | **PASS** [corrected 2026-09-30: UNEVIDENCED. No CI run found.] | ## Next: operations & dogfood @@ -35,5 +39,29 @@ See [Ops readiness v1](OpsReadiness-v1.md). Full stack: `make ashlar-ready-gate` ## Sign-off -- [x] `ship-gate-full` green (2026-05-19) -- [x] Composition + mesh gates green same week +- [x] `ship-gate-full` green (2026-05-19) [corrected 2026-09-30: Tiers B–D are UNEVIDENCED; see + [Corrections](#corrections-2026-09-30).] +- [x] Composition + mesh gates green same week [corrected 2026-09-30: SUPPORTED, carried forward: + Composition Mesh Gate run 26262666838 (`ee8befa7` → `0ecb5bfc`, identical tree `f1e22dfd`) + and Mesh virtual lab gate run 26129468841 (`7457e6c5` → `0ecb5bfc`); see + [Composition & mesh corrections](CompositionMeshReadiness-v1.md#corrections-2026-09-30).] + +## Corrections (2026-09-30) + +**Evidence rule from 2026-09-30.** A row may read PASS only if it names a run id or check run on +the SHA in this sheet's header. Any other row reads NOT RUN, or CARRIED FORWARD with both SHAs. A +readiness PASS must cite a `Readiness summary` run whose verdict is `verified`. The full rule and +the four labels are in the [production-readiness README](README.md#evidence-rule-for-readiness-records-added-2026-09-30). + +**Which SHA.** This sheet names none. It was committed in `0ecb5bfc` (#110, 2026-05-22 01:17 UTC), +whose tree is identical to PR #110's head `ee8befa7`. No commit contained the ship-gate scripts on +2026-05-19. At `0ecb5bfc`, `.github/workflows/ship-gate.yml` triggers only on `workflow_dispatch`. +Its first run was 31972468094 on 2026-08-16, and there are 117 runs in all. The workflow has +no May run of any tier. + +| Row | Label | Evidence | +|-----|-------|----------| +| Tier A | SUPPORTED, carried forward `7457e6c5` → `0ecb5bfc` | The tier script says it mirrors Production Readiness Gate v1. Run 26129468793 of that workflow (push, `7457e6c5`, 2026-05-19) passed on ubuntu, macOS and windows. It ran the same three builds, the same two host DI smoke tests, the same CLI pipeline validate/run/fallback and the same LiteDB cross-process resume. Outside docs, gate scripts, workflows and the Makefile, `7457e6c5` and `0ecb5bfc` differ only in global.json, two helper scripts and one test file. | +| Tier B | UNEVIDENCED | No CI run found. At `0ecb5bfc`, `make test-prod-style` selects 68 test methods and BaseFrameworkSmokeTests selects 9. The `doctor --json` leg fails the tier only when `SHIP_GATE_STRICT_DOCTOR=1`. | +| Tier C | UNEVIDENCED | No CI run of the local release preflight was found. | +| Tier D | UNEVIDENCED | No CI run found. In the default profile the sheet records, this tier is `doctor --json`; the release bundle runs only with `SHIP_GATE_RUN_RUNTIME_GATE=1`. | diff --git a/docs/specs/SPEC-006-keys-and-signing.md b/docs/specs/SPEC-006-keys-and-signing.md index d13e083e7..e15bdeead 100644 --- a/docs/specs/SPEC-006-keys-and-signing.md +++ b/docs/specs/SPEC-006-keys-and-signing.md @@ -258,9 +258,10 @@ Rules: `src/Ashlar.Tests.Kernel/**` - so editing `GateStore` or `AdmissionGate` re-runs the 412 cases that guard them. It did not before. A diff that touches none of those entries still skips the lanes, which is the design; it is not evidence about this rule either way. And the sweep emits no - per-project count, while a project that selects zero tests still passes - (`ValidationServiceAdapter.cs:248`), so a green sweep does not by itself prove these facts - executed. [corrected 2026-09-29; routing 2026-09-30] + per-project count. Since 2026-09-30 an unfiltered sweep fails any discovered project that + executes zero tests (`ValidationServiceAdapter.ZeroExecutionError`), but that floor is one + executed test, not 412, so a green sweep still does not by itself prove these facts executed. + [corrected 2026-09-29; routing 2026-09-30; zero-execution floor 2026-09-30] *Residuals, recorded rather than papered over. Every anchor this rule rests on is a file in the state root, and each residual below is demonstrated by a fact in `GateSignatureResidualTests` diff --git a/scripts/application-gate-tier-c.sh b/scripts/application-gate-tier-c.sh index e1c2fac81..1ff939c5e 100755 --- a/scripts/application-gate-tier-c.sh +++ b/scripts/application-gate-tier-c.sh @@ -5,12 +5,22 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT" INFRA="src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj" +TRX_DIR="test-results/application-gate-tier-c" +rm -rf "$TRX_DIR" echo "== Application Tier C: in-process API (DI + HTTP demos) ==" -dotnet build "$INFRA" -f net8.0 -v minimal -ASHLAR_ALLOW_MOCK=1 dotnet test "$INFRA" -f net8.0 --no-build \ +# net10.0, not net8.0: both classes link Ashlar.API, which ships on net10.0 only, and the csproj +# compiles Tests/API and Tests/VirtualProduction out of the net8.0 leg. At -f net8.0 this lane +# selected ZERO tests and printed PASS - FrameworkVirtualProdDemosTests from the day it was written +# (VirtualProduction was already net10.0-only), ApiDevelopmentHostDiTests since e6b5579f +# (2026-08-17). Measured in the dev container at 876cc527: net8.0 selects 0, net10.0 selects 4. +# 720s because ApiDevelopmentHostDiTests carries Timeout = TestTimeouts.HostTouching (480s). +dotnet build "$INFRA" -f net10.0 -v minimal +ASHLAR_ALLOW_MOCK=1 dotnet test "$INFRA" -f net10.0 --no-build \ --filter "FullyQualifiedName~ApiDevelopmentHostDiTests|FullyQualifiedName~FrameworkVirtualProdDemosTests" \ - --blame-hang-timeout 180s --blame-hang-dump-type none + --logger "trx;LogFileName=in-process-api.trx" --results-directory "$TRX_DIR" \ + --blame-hang-timeout 720s --blame-hang-dump-type none +bash scripts/ci/zero-test-guard.sh "$TRX_DIR/in-process-api.trx" echo "" echo "application-gate-tier-c: PASS" diff --git a/scripts/cert-gate-config.sh b/scripts/cert-gate-config.sh index 9a4230d27..18db85faa 100644 --- a/scripts/cert-gate-config.sh +++ b/scripts/cert-gate-config.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Single source of truth for cert-gate test filter and expected count. +# Single source of truth for cert-gate test filter and expected count, and for reading TRX counters. set -euo pipefail # Must match tests exercised by scripts/run-cert-gate.sh and .github/workflows/cert-gate.yml @@ -27,3 +27,34 @@ cert_gate_expected_count() { local root="${1:?repo root required}" cert_gate_list_tests "${root}" | wc -l | tr -d '[:space:]' } + +# Prints one numeric attribute (total, executed, passed, ...) of the TRX's element, or +# fails. It reads ONLY that element. Test output is written into the TRX before the run summary, +# so the first `total="N"` in the file is not necessarily the counter - but XML escapes '<' in text +# and in attribute values alike, so a literal ' element carrying +# exactly one numeric value for the attribute. Callers assign with a plain `var="$(...)"`, never +# `local var="$(...)"`, so the failure is not masked and `set -e` stops them. +cert_gate_trx_counter() { + local trx="${1:?trx path required}" + local name="${2:?counter name required}" + local counters found value + counters="$(tr '\r\n' ' ' < "${trx}" | grep -oE ']*>' || true)" + found="$(printf '%s\n' "${counters}" | grep -c ' element in ${trx}, found ${found}" >&2 + return 1 + fi + value="$(printf '%s\n' "${counters}" | grep -oE "[[:space:]]${name}=\"[0-9]+\"" | sed -E 's/.*="([0-9]+)"$/\1/' || true)" + if [[ ! "${value}" =~ ^[0-9]+$ ]]; then + echo "cert-gate: the element in ${trx} has no single numeric '${name}' attribute" >&2 + return 1 + fi + printf '%s\n' "${value}" +} diff --git a/scripts/cert-gate-skip-guard.sh b/scripts/cert-gate-skip-guard.sh new file mode 100755 index 000000000..f5ce32289 --- /dev/null +++ b/scripts/cert-gate-skip-guard.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# Fail when the number of SKIPPED cert-gate tests differs from the committed baseline, in either +# direction. +# +# WHY. `dotnet test` exits 0 when a test is skipped, and the TRX counts a skipped test in its +# `total`. cert-gate-zero-test-guard.sh floors that `total` against `dotnet test --list-tests`, +# which lists a skipped test as well, so a skip moves both sides of the floor together. Measured +# 2026-09-30 by putting [Fact(Skip = "...")] on one Certification test: run-cert-gate.sh exited 0 +# and the zero-test guard printed "cert-gate reported 1288 tests (expected>=1283, derived from +# --list-tests).", while the TRX read total=1288 executed=1287. +# +# WHAT. `total - executed` from the TRX element - the results the run reports but did +# not execute - must EQUAL the integer in scripts/cert-gate-skipped.baseline. Not the `notExecuted` +# attribute: in that same measured run it read 0 beside the skipped result, so it cannot see a skip. +# more skipped than the baseline: red, naming the skipped tests. If the skip is deliberate, raise +# the baseline in the same change, so it is a reviewed diff. +# fewer skipped than the baseline: red too. Lower the baseline, so slack left behind cannot absorb +# a later skip without anyone seeing it. +# +# FAILS CLOSED. A missing TRX, a missing or malformed baseline, or a TRX without exactly one readable +# element is a failure, never a pass. +# +# NOT COVERED. The count is what is enforced, so un-skipping one test and skipping another in the +# same change keeps it equal; both edits are still visible in that change's C# diff. Test failures +# are not this guard's business - `executed` includes them and run-cert-gate.sh exits with the +# `dotnet test` status. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/cert-gate-config.sh +source "${ROOT}/scripts/cert-gate-config.sh" + +TRX="${1:-test-results/cert-gate.trx}" +BASELINE_REL="scripts/cert-gate-skipped.baseline" +BASELINE_FILE="${ROOT}/${BASELINE_REL}" + +if [[ ! -f "${TRX}" ]]; then + echo "cert-gate skip guard: TRX not found: ${TRX}" + exit 1 +fi + +if [[ ! -f "${BASELINE_FILE}" ]]; then + echo "cert-gate skip guard: ${BASELINE_REL} is missing, so the skipped-test count cannot be checked." + exit 1 +fi + +# One integer; blank lines and lines starting with # are ignored. Trimming trailing whitespace also +# removes the CR a Windows checkout leaves (the file is text=auto, so CRLF there). +BASELINE_RAW="$(sed -E 's/^[[:space:]]+//; s/[[:space:]]+$//' < "${BASELINE_FILE}" | grep -vE '^(#|$)' || true)" +if [[ ! "${BASELINE_RAW}" =~ ^[0-9]+$ ]]; then + echo "cert-gate skip guard: ${BASELINE_REL} must hold exactly one non-negative integer (lines starting with # are comments)." + echo " found: ${BASELINE_RAW:-}" + exit 1 +fi +BASELINE=$((10#${BASELINE_RAW})) + +TOTAL="$(cert_gate_trx_counter "${TRX}" total)" +EXECUTED="$(cert_gate_trx_counter "${TRX}" executed)" + +if (( EXECUTED > TOTAL )); then + echo "cert-gate skip guard: the TRX says executed=${EXECUTED} but total=${TOTAL}; it cannot be read as a run summary." + exit 1 +fi + +SKIPPED=$((TOTAL - EXECUTED)) + +# The names are for the reader. The count above is what is enforced. +skipped_names() { + tr '\r\n' ' ' < "${TRX}" \ + | grep -oE ']*outcome="NotExecuted"[^>]*>' \ + | sed -E 's/.*[[:space:]]testName="([^"]*)".*/\1/' \ + | sed -e 's/"/"/g' -e "s/'/'/g" -e 's/<//g' -e 's/&/\&/g' \ + | sort || true +} + +print_skipped() { + local names + names="$(skipped_names)" + if [[ -n "${names}" ]]; then + printf '%s\n' "${names}" | sed 's/^/ /' + else + echo " (no result in the TRX is marked NotExecuted; the counters alone say ${SKIPPED})" + fi +} + +if (( SKIPPED > BASELINE )); then + echo "cert-gate skip guard: ${SKIPPED} cert-gate test(s) were SKIPPED; ${BASELINE_REL} records ${BASELINE}." + echo " total=${TOTAL} executed=${EXECUTED}. dotnet test reports a skipped test as success, so a skip" + echo " on a cert-gate test is a merge-blocking assertion that no longer blocks anything." + echo " If the skip is deliberate, set ${BASELINE_REL} to ${SKIPPED} in the same change so the" + echo " decision is reviewed. Otherwise remove the Skip. Skipped in this run:" + print_skipped + exit 1 +fi + +if (( SKIPPED < BASELINE )); then + echo "cert-gate skip guard: only ${SKIPPED} cert-gate test(s) were skipped; ${BASELINE_REL} records ${BASELINE}." + echo " total=${TOTAL} executed=${EXECUTED}. Lower ${BASELINE_REL} to ${SKIPPED} in this change." + echo " Slack left in the baseline is room for a later skip to land without anyone seeing it." + exit 1 +fi + +echo "cert-gate skip guard: ${SKIPPED} skipped (total=${TOTAL} executed=${EXECUTED}), matching ${BASELINE_REL}." +if (( SKIPPED > 0 )); then + echo " Skipped by recorded decision:" + print_skipped +fi diff --git a/scripts/cert-gate-skipped.baseline b/scripts/cert-gate-skipped.baseline new file mode 100644 index 000000000..ab7601f73 --- /dev/null +++ b/scripts/cert-gate-skipped.baseline @@ -0,0 +1,15 @@ +# How many cert-gate tests are SKIPPED on a clean run: the TRX total minus executed. +# +# scripts/cert-gate-skip-guard.sh (run by scripts/run-cert-gate.sh, i.e. by the required cert-gate +# check) refuses a run whose count differs from this number in EITHER direction: +# - more skipped than this: a merge-blocking test stopped running and dotnet test still exited 0. +# If that is deliberate, raise this number in the same change so the decision is reviewed. +# - fewer skipped than this: lower it, so slack cannot quietly absorb a later skip. +# +# Measured 2026-09-30: 1288 total, 1288 executed, 0 skipped - in the devtest container and in the +# cert-gate logs of ten CI runs (master and pull_request) from 2026-09-29/30. No test in the +# cert-gate selection declares a Skip, and xunit 2.x has no runtime skip. +# +# It is its own committed file, not a number derived from the run or read from a generated +# artifact, because a baseline that is regenerated ratchets against itself and enforces nothing. +0 diff --git a/scripts/cert-gate-summary.sh b/scripts/cert-gate-summary.sh index 0ba742468..6112380c2 100755 --- a/scripts/cert-gate-summary.sh +++ b/scripts/cert-gate-summary.sh @@ -1,7 +1,19 @@ #!/usr/bin/env bash # Write generation-safety results to GITHUB_STEP_SUMMARY (or stdout locally). +# +# Exits 1 when there is nothing to summarise - no TRX, or a TRX whose cannot be read. It +# used to exit 0 with a "Tests may not have run" note, which painted a green step beside a run that +# produced no results. Every run that reaches this without a TRX has already failed in +# run-cert-gate.sh (its guards refuse a missing TRX), so exiting 1 here changes no verdict; it only +# stops this step contradicting it. +# +# Informational otherwise: the gate itself is run-cert-gate.sh and its guards, not this table. set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/cert-gate-config.sh +source "${ROOT}/scripts/cert-gate-config.sh" + TRX="${1:-test-results/cert-gate.trx}" SUMMARY="${GITHUB_STEP_SUMMARY:-/dev/stdout}" @@ -9,9 +21,10 @@ if [[ ! -f "${TRX}" ]]; then { echo "## cert-gate — generation safety" echo "" - echo "TRX not found (\`${TRX}\`). Tests may not have run." + echo "TRX not found (\`${TRX}\`). No results to report: the tests did not run, or ran and wrote nothing." } >> "${SUMMARY}" - exit 0 + echo "cert-gate summary: TRX not found: ${TRX}" >&2 + exit 1 fi outcome_for() { @@ -31,8 +44,10 @@ outcome_for() { fi } -EXECUTED="$(grep -oE 'executed="[0-9]+"' "${TRX}" | head -1 | sed 's/executed="//;s/"//')" -EXECUTED="${EXECUTED:-0}" +# Plain assignments, so an unreadable stops the script under `set -e`. +TOTAL="$(cert_gate_trx_counter "${TRX}" total)" +EXECUTED="$(cert_gate_trx_counter "${TRX}" executed)" +SKIPPED=$((TOTAL - EXECUTED)) { echo "## cert-gate — generation safety" @@ -44,5 +59,6 @@ EXECUTED="${EXECUTED:-0}" echo "| \`GoodGeneration_WeakWitness_Rejects_WithTeeth\` | $(outcome_for "GoodGeneration_WeakWitness_Rejects_WithTeeth") |" echo "| \`DependencyLeakGeneration_Rejects_DependencyCheck\` | $(outcome_for "DependencyLeakGeneration_Rejects_DependencyCheck") |" echo "" - echo "Total cert-gate tests executed: **${EXECUTED}** (expected ≥ derived count from cert-gate filter)." + echo "cert-gate tests: **${EXECUTED}** executed of **${TOTAL}** reported, **${SKIPPED}** skipped." + echo "Reported must be ≥ the \`--list-tests\` count for the cert-gate filter, and skipped must equal \`scripts/cert-gate-skipped.baseline\`." } >> "${SUMMARY}" diff --git a/scripts/cert-gate-zero-test-guard.sh b/scripts/cert-gate-zero-test-guard.sh index 327c3243c..9b260e16b 100755 --- a/scripts/cert-gate-zero-test-guard.sh +++ b/scripts/cert-gate-zero-test-guard.sh @@ -19,8 +19,12 @@ if [[ -z "${MIN_EXPECTED}" || "${MIN_EXPECTED}" -lt 1 ]]; then exit 1 fi -REPORTED="$(grep -oE 'total="[0-9]+"' "${TRX}" | head -1 | sed 's/total="//;s/"//')" -REPORTED="${REPORTED:-0}" +# `total` counts every result the TRX holds, SKIPPED ones included, and --list-tests lists a skipped +# test too - so a skip moves both sides of this floor together and it cannot see one. That is not +# this guard's job: cert-gate-skip-guard.sh pins total - executed to a committed baseline, and +# run-cert-gate.sh runs it straight after this one. Read from the element, not the first +# `total="N"` in the file, which can be text a test printed (see cert_gate_trx_counter). +REPORTED="$(cert_gate_trx_counter "${TRX}" total)" if [[ "${REPORTED}" -lt "${MIN_EXPECTED}" ]]; then echo "cert-gate matched fewer tests than expected (reported=${REPORTED}, expected>=${MIN_EXPECTED}) — filter is stale." diff --git a/scripts/ci/verify-zero-test-guard-wiring.py b/scripts/ci/verify-zero-test-guard-wiring.py new file mode 100644 index 000000000..5c90bb3ee --- /dev/null +++ b/scripts/ci/verify-zero-test-guard-wiring.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python3 +"""Every pinned `dotnet test --filter` is followed by scripts/ci/zero-test-guard.sh over its own TRX. + +WHY. VSTest exits 0 when a --filter selects nothing. scripts/ci/zero-test-guard.sh turns that into a +red lane, but only where it is called: a pinned filter added tomorrow without it is exactly as quiet +as the three that sat green on zero tests until this check existed (security-gate Tier B, +compat-gate Tier A step 1, application-gate Tier C). So the population is DERIVED, not listed: +every `dotnet test` in + - .github/workflows/*.yml + - scripts/**/*.sh and tests/**/*.sh (tests/uat/*.sh is run by uat-gate.yml) + - the Makefile + - deploy/compose/*.yml (compose-gate.yml brings up docker-compose.test.yml, whose service command + runs a pinned filter; a folded `command: >` scalar is read as the one line the shell sees) +except the files in FIXTURE_TEXT, whose `dotnet test` lines are this check's own test inputs. + +A filter is PINNED when it has at least one positive term (`=` or `~`, not `!=` / `!~`): it names +what it selects, so it can go stale. An exclusion-only filter (`FullyQualifiedName!~X`) selects "the +rest of the project" and is out of scope, as is a run with no filter at all. A filter this check +cannot resolve to text (a variable it cannot see) counts as pinned: the unknown direction is the one +that must not pass quietly. + +For each pinned invocation it requires: + 1. a TRX of its own: `--logger "trx;LogFileName="`, the name unique in the file; + 2. `-f`/`--framework` when the project targets more than one framework - one invocation runs every + target and each overwrites the previous TRX ("WARNING: Overwriting results file"), so a target + that matched nothing leaves no trace in the file that survives (measured on the security-gate + Tier B filter: the surviving net10.0 TRX said 44 executed while net8.0 had matched nothing); + 3. for a .sln/.slnf target, `LogFilePrefix=

` instead (one TRX per project and framework) and a + guard call in `--solution` mode naming

; + 4. a LATER guard call naming that TRX, in the SAME unit that ran the test - the same Make target, + the same workflow `run:` block, the same script. A guard in another Make target or another step + runs in a different shell, possibly never. For a pinned run in a compose file, the unit is the + workflow/script/Make `run` that brings the file up (`docker compose -f ... up`): the guard + must follow that line, because that is where the TRX lands on the host. + "Naming" is read from the guard's ARGUMENTS, not from its line: the words after the guard's path, + quotes removed, stopping at a word that starts with `#` (a trailing comment names nothing). A + GitHub expression is compared as written on both sides (`${{ matrix.os }}` equals + `${{matrix.os}}` and nothing else). One argument's basename must EQUAL the LogFileName - so + `out/barfoo.trx` does not name foo.trx, and neither does `out/other.trx # foo.trx`. In + `--solution` mode one argument's basename must match `

_*` or `*__

_*` for LogFilePrefix

. + Shell variables are not expanded: the TRX name must be written literally in the argument; + 5. that the guard call can fail its unit. It is refused when + - it is not a command of its own: the line must start with `bash /scripts/ci/zero-test-guard.sh` + (so not `echo ...`, `if ...`, `! ...`). `scripts/cert-gate-zero-test-guard.sh` is not this guard; + - it is text inside a heredoc body (`cat > x <<'EOF'` ... `EOF`): that is data for whatever reads + the heredoc, not a command this unit runs; + - it shares its line with a control operator - `||` (`|| true`, `|| echo ...`), `&&`, `;`, `|` + or `&`. Each lets the line succeed while the guard fails: `x || y` runs y, `x && y` and `x; y` + are exempt from `set -e` or report y's status, a pipeline reports its last command, and `&` + reports nothing. A guard needs a line of its own; + - a Make recipe line carries the `-` prefix (make prints "Error 1 (ignored)" and carries on); + - `set -e` is not in force where it runs: in a script, the last `set -e`/`set +e` before it must + be `-e`; in a workflow `run:` block (GitHub runs bash with -e) no `set +e` may precede it; + 6. that nothing between the test and its guard, in the same script or run block, ends the unit with + success first: an `exit` or `return` in command position whose status is absent (`exit`: the + last command's, which under set -e is the test's 0) or a literal 0 (`exit 0`, `exit "0"`). Lines + inside heredoc bodies are not commands and are skipped. A Make recipe is exempt: each recipe + line runs in a shell of its own, so an `exit 0` line ends only itself. + +What it does NOT check - each of these would still let a guarded lane go quiet: + - that the step or job is not `continue-on-error`, or that its `if:` ever lets it run, or that the + lane runs on any trigger at all. It pins the wiring, not the reachability; + - the step's shell. A `shell: pwsh` step (or a Windows runner's default) does not stop on a failing + command mid-script. Every guarded step today declares `shell: bash` or runs on ubuntu; + - control flow. A unit is read as a flat list of lines, not as a graph: a guard in a DIFFERENT + branch of an if/else (or case arm) from its test is accepted, although it never runs after that + test. So is a guard inside a shell function (never called, or called before the test), a subshell + or a loop, and a `set +e` inside a function; + - an `exit`/`return` between the test and its guard whose status is not a literal: `exit "$rc"`, + `exit $?`. perf-certification.yml re-raises `exit "$rc"` inside `if [[ "$rc" -ne 0 ]]`, which is + correct, and this check cannot tell it from `rc=0; exit "$rc"`. Nor does it see `exec`, `kill $$`, + or an `exit 0` inside a function that is called between the two; + - a heredoc it does not recognise as one: its delimiter must be a plain word (`EOF`, `'EOF'`, + `"EOF"`, `\\EOF`, with or without `-`). A `dotnet test` inside a heredoc body IS still counted as a + pinned run (the heredoc may be a script that is run), and a guard next to it inside the same body + is refused - the fail-closed direction. + +Callers it does NOT reach (a pinned filter there is invisible to it): + - Dockerfile CMD lines (.docker/Dockerfile.test-*). Measured when this was written: no workflow runs + one of those images with its default command - compose-gate.yml overrides it with the compose + service's `command:`, test-trust-multi-env.yml and full-platform-readiness-gate.yml with + `docker run bash -c ...`; + - C# that launches `dotnet test` (application/src/Ashlar.CLI/Commands/*, src/Ashlar.Infrastructure/**); + - PowerShell (scripts/test-in-container.ps1, a developer harness) and python scripts; + - a filter a workflow passes in through an input, an env var or a matrix value the scan cannot see + (it counts as pinned and unresolvable, so it is refused rather than missed - but only if the + `dotnet test` itself is in a scanned file). + +Run: python3 scripts/ci/verify-zero-test-guard-wiring.py # the repository (all checks) + python3 scripts/ci/verify-zero-test-guard-wiring.py --root DIR # a fixture tree (no floor, + # no required lanes, no stale- + # exemption check) +""" +import collections +import fnmatch +import os +import re +import sys + +NL = chr(10) +CR = chr(13) +TAB = chr(9) +BS = chr(92) + +GUARD_PATH = "scripts/ci/zero-test-guard.sh" + +# Whole files or Make targets this check does not hold to the rule, each with its reason. Every entry +# must still match at least one pinned invocation on the real tree, so the list cannot rot quietly. +EXEMPT_FILES = [ + ("scripts/test-in-container.sh", + "developer harness: forwards whatever filter the caller typed, runs no gate"), + ("scripts/cert-gate-*.sh", + "cert-gate's own scripts: cert-gate-config.sh derives the expected count with --list-tests " + "(not a test run), and cert-gate has its own fail-closed guard, cert-gate-zero-test-guard.sh"), + ("scripts/run-cert-gate.sh", + "cert-gate's runner, guarded by scripts/cert-gate-zero-test-guard.sh"), + ("scripts/validate-safe.sh", + "local-only helper that no workflow reaches"), + (".github/workflows/full-platform-readiness-gate.yml", + "owned by the readiness gate's own change; its smoke lanes are not yet guarded"), + ("tests/uat/tier0-2.sh", + "uat-gate Tier 2 collects verdicts instead of exiting on the first failure, and it already FAILs " + "unless the run log reports Total=16 (the count TesterQuickstart claims) and when no total can be " + "read - an exact count, stronger than this guard, though Total also counts skipped tests. Its " + "second pinned call is --list-tests, which runs nothing"), +] +EXEMPT_MAKE_TARGETS = [ + ("test-all-platforms", "local-only Docker matrix that no workflow reaches"), + ("test-prime-time", "local-only solution-filter sweep that no workflow reaches"), + ("test-mesh-lab", "local-only Docker mesh-lab run that no workflow reaches"), +] +# Files whose `dotnet test` lines are TEXT, not lanes. They are not counted as pinned invocations at +# all (so they cannot pad the floor below), and each must still match one, like an exemption. +FIXTURE_TEXT = [ + ("tests/scripts/zero-test-guard-wiring.test.sh", + "this check's own tests: their heredocs are the shapes it must refuse or accept, written into " + "throwaway trees"), +] + +# The real tree must yield at least this many pinned invocations (fixture text excluded). A scanner +# that stops matching finds nothing and reports clean; this is the floor under that. It is a literal +# on purpose - lowering it is a visible diff, never a side effect of regenerating something. Set +# below the count at the time of writing: 106 (91 guarded, 15 exempt). Losing the workflows (37), the +# Makefile (34) or scripts/ (28) from the scan takes it below the floor. +MINIMUM_PINNED = 100 + +# Lanes this check must find wired on the real tree: the two this change repaired, and one per other +# kind of file it scans (a workflow, the Makefile, a compose file). Losing any from the scan is how +# the population rots back into a list. tests/**/*.sh is held by its exemption's stale check. +REQUIRED_WIRED = [ + ("scripts/security-gate-tier-b.sh", "api-security.trx"), + ("scripts/compat-gate-tier-a.sh", "mesh-checkpoint-migration.trx"), + (".github/workflows/mcp-a2a-gate.yml", "protocol-prodstyle.trx"), + ("Makefile", "kernel-gate-hosting.trx"), + ("deploy/compose/docker-compose.test.yml", "ubuntu-baseframework.trx"), +] + +# The test images COPY the repository to /workspace (.docker/Dockerfile.test-caching*), so a project +# path under it is the same project in this tree. +CONTAINER_WORKSPACE = "/workspace/" + +FILTER = re.compile(r"""--filter\s+(?:"([^"]*)"|'([^']*)'|(\S+))""") +FRAMEWORK = re.compile(r"""(?:^|\s)(?:-f|--framework)\s+["']?([^\s"']+)""") +# A GitHub expression inside a name (`smoke-${{ matrix.os }}.trx`) carries spaces, so it is one unit. +LOG_NAME = re.compile(r"LogFileName=((?:\$\{\{[^}]*\}\}|[^;\"'\s])+)") +LOG_PREFIX = re.compile(r"LogFilePrefix=((?:\$\{\{[^}]*\}\}|[^;\"'\s])+)") +SHELL_ASSIGN = re.compile(r"""^\s*(?:export\s+|readonly\s+)?([A-Za-z_][A-Za-z0-9_]*)=(?:"([^"]*)"|'([^']*)'|(\S*))""") +MAKE_ASSIGN = re.compile(r"^([A-Za-z_][A-Za-z0-9_]*)\s*(?::=|\?=|=)\s*(.*?)\s*$") +MAKE_TARGET = re.compile(r"^([A-Za-z0-9_.%-]+)\s*:(?!=)") +VAR_REF = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}|\$\(([A-Za-z_][A-Za-z0-9_]*)\)|\$([A-Za-z_][A-Za-z0-9_]*)") +# `key: >` (a YAML folded scalar): its lines are one line to whatever reads the value. +FOLDED_KEY = re.compile(r"^(\s*(?:-\s+)?)[A-Za-z0-9_.-]+:\s*>[-+0-9]*\s*$") +RUN_KEY = re.compile(r"^(\s*(?:-\s+)?)run:\s*(.*)$") +BLOCK_SCALAR = re.compile(r"^[|>][-+0-9]*\s*(?:#.*)?$") +# The guard must be the command itself: `bash /scripts/ci/zero-test-guard.sh`, quoted or not. +GUARD_COMMAND = re.compile(r"""^bash\s+["']?(?:[^\s"']*/)?scripts/ci/zero-test-guard\.sh["']?(?:\s|$)""") +CONTROL = re.compile(r"\|\||&&|[|&;]") +SET_E = re.compile(r"(?:^|[\s;])set\s+(?:-[A-Za-z]*e[A-Za-z]*|-o\s+errexit)(?![A-Za-z])") +SET_NO_E = re.compile(r"(?:^|[\s;])set\s+(?:\+[A-Za-z]*e[A-Za-z]*|\+o\s+errexit)(?![A-Za-z])") +COMPOSE_CALL = re.compile(r"docker[ -]compose\s(.*)") +COMPOSE_FILE_ARG = re.compile(r"""(?:^|\s)(?:-f|--file)\s+["']?([^\s"']+)""") +COMPOSE_UP = re.compile(r"(?:^|\s)up(?:\s|$)") +# A heredoc operator (not `<<<`, a here-string) and the plain-word delimiter after it. +HEREDOC_OP = re.compile(r"(?" + +Guard = collections.namedtuple("Guard", "idx text block defect args") + + +def rel(root, path): + return os.path.relpath(path, root).replace(BS, "/") + + +def kind_of(name): + if name == "Makefile": + return "make" + if name.startswith(".github/workflows/"): + return "workflow" + if name.startswith("deploy/compose/"): + return "compose" + return "shell" + + +def files_to_scan(root): + out = [] + make = os.path.join(root, "Makefile") + if os.path.isfile(make): + out.append(make) + for sub in ("scripts", "tests"): + base = os.path.join(root, sub) + for d, _, fs in os.walk(base): + out += [os.path.join(d, f) for f in fs if f.endswith(".sh")] + for sub, exts in ((os.path.join(".github", "workflows"), (".yml", ".yaml")), + (os.path.join("deploy", "compose"), (".yml", ".yaml"))): + d = os.path.join(root, sub) + if os.path.isdir(d): + out += [os.path.join(d, f) for f in os.listdir(d) if f.endswith(exts)] + return sorted(out) + + +def indent_of(line): + return len(line) - len(line.lstrip(" ")) + + +def logical_lines(text, folded=False): + """Joins backslash-continued lines, and (folded=True) a YAML `key: >` scalar's lines. + + Returns [(first_line_number, text, raw_first_line)].""" + lines = text.replace(CR + NL, NL).split(NL) + out, buf, start, i = [], [], None, 0 + while i < len(lines): + line = lines[i] + if folded and start is None: + m = FOLDED_KEY.match(line) + if m: + key_indent, j, body = len(m.group(1)), i + 1, [] + while j < len(lines) and (not lines[j].strip() or indent_of(lines[j]) > key_indent): + if lines[j].strip(): + body.append(lines[j].strip()) + j += 1 + out.append((i + 1, " ".join(body), line)) + i = j + continue + if start is None: + start = i + 1 + stripped = line.rstrip() + i += 1 + if stripped.endswith(BS): + buf.append(stripped[:-1]) + continue + buf.append(line) + out.append((start, " ".join(s.strip() for s in buf), lines[start - 1])) + buf, start = [], None + if buf: + out.append((start, " ".join(s.strip() for s in buf), lines[start - 1])) + return out + + +def run_blocks(text): + """Maps each physical line number of a workflow to the `run:` block it belongs to (or None).""" + lines = text.replace(CR + NL, NL).split(NL) + block, cur, cur_indent = {}, None, None + for n, line in enumerate(lines, 1): + if cur is not None: + if not line.strip() or indent_of(line) > cur_indent: + block[n] = cur + continue + cur = None + m = RUN_KEY.match(line) + if m: + block[n] = n + if BLOCK_SCALAR.match(m.group(2).strip()): + cur, cur_indent = n, len(m.group(1)) + return block + + +def is_comment(raw): + return raw.lstrip().startswith("#") + + +def variables(text, is_make): + found = {} + for line in text.replace(CR + NL, NL).split(NL): + if is_make: + if line.startswith(TAB): + continue + m = MAKE_ASSIGN.match(line) + if m and m.group(1) not in found: + found[m.group(1)] = m.group(2) + else: + m = SHELL_ASSIGN.match(line) + if m and m.group(1) not in found: + found[m.group(1)] = next(g for g in m.groups()[1:] if g is not None) + return found + + +def resolve(token, found): + for _ in range(4): + new = VAR_REF.sub(lambda m: found.get(m.group(1) or m.group(2) or m.group(3), m.group(0)), token) + if new == token: + break + token = new + return token.strip("\"'") + + +def is_pinned(flt): + if flt is None: + return False + if "$" in flt: + return True # unresolvable: fail closed + return re.search(r"(?([^<]*)", text) + return bool(m and ";" in m.group(1).strip().strip(";")) + + +def unquoted(cmd): + """The command with quoted text, GitHub expressions and fd redirections blanked out.""" + cmd = re.sub(r"\$\{\{[^}]*\}\}", "X", cmd) + cmd = re.sub(r'"(?:[^"\\]|\\.)*"', '""', cmd) + cmd = re.sub(r"'[^']*'", "''", cmd) + return re.sub(r"\d*[<>]&\d*-?|&>>?", " ", cmd) + + +def expr_norm(s): + """GitHub expressions with their inner whitespace removed: `${{ matrix.os }}` becomes one word that + equals itself however it was spaced, and nothing else (not `${{ matrix.label }}`).""" + return GH_EXPR.sub(lambda m: "${{" + re.sub(r"\s+", "", m.group(1)) + "}}", s) + + +def masked(line): + """The line with the inside of every quoted string replaced by `x` and a trailing comment cut off. + Offsets are unchanged, so a match found in it is at the same place in the line, and nothing it + finds was inside quotes.""" + out, i, n = list(line), 0, len(line) + while i < n: + c = line[i] + if c == BS: + i += 2 + continue + if c == "'": + j = line.find("'", i + 1) + elif c == '"': + j = i + 1 + while j < n and line[j] != '"': + j += 2 if line[j] == BS else 1 + else: + i += 1 + continue + j = n if j < 0 or j > n else j + for k in range(i + 1, j): + out[k] = "x" + i = j + 1 + text = "".join(out) + m = re.search(r"(?:^|\s)#", text) + return text[:m.start()] if m else text + + +def words(cmd): + """The shell words of one command line: quotes removed; an unquoted `;`, `&`, `|`, `<` or `>` ends + a word and is a word of its own; a `#` that starts a word starts a comment, which ends the line.""" + out, cur, inword, i, n = [], [], False, 0, len(cmd) + while i < n: + c = cmd[i] + if c in (" ", TAB) or c in WORD_BREAK: + if inword: + out.append("".join(cur)) + cur, inword = [], False + if c in WORD_BREAK: + out.append(c) + i += 1 + continue + if c == "#" and not inword: + break + inword = True + if c == "'": + j = cmd.find("'", i + 1) + j = n if j < 0 else j + cur.append(cmd[i + 1:j]) + i = j + 1 + elif c == '"': + j = i + 1 + while j < n and cmd[j] != '"': + if cmd[j] == BS and j + 1 < n: + j += 1 + cur.append(cmd[j]) + j += 1 + i = j + 1 + elif c == BS and i + 1 < n: + cur.append(cmd[i + 1]) + i += 2 + else: + cur.append(c) + i += 1 + if inword: + out.append("".join(cur)) + return out + + +def command_of(kind, text, raw): + """(command, make_prefix): the line without a workflow's `run:` key or a Make recipe's @+- prefix.""" + cmd = text.strip() + if kind == "workflow": + cmd = re.sub(r"^(?:-\s+)?run:\s*", "", cmd) + prefix = "" + if kind == "make" and raw.startswith(TAB): + prefix = re.match(r"^[@+\-\s]*", cmd).group(0) + cmd = cmd[len(prefix):] + return cmd, prefix + + +def guard_args(kind, text, raw): + """The words after the guard's path on its line; [] when no word is the guard's path.""" + toks = words(expr_norm(command_of(kind, text, raw)[0])) + for _ in range(3): + for i, t in enumerate(toks): + if t == GUARD_PATH or t.endswith("/" + GUARD_PATH): + return toks[i + 1:] + # `echo "bash scripts/ci/zero-test-guard.sh out/x.trx"`: the call sits inside one word. Read + # what it names there as well, so the refusal says this guard cannot fail, not that none exists. + inner = next((t for t in toks if GUARD_PATH in t), None) + if inner is None: + return [] + toks = words(inner) + return [] + + +def names_trx(args, key, mode): + """Whether one of a guard's arguments names this run's TRX: its basename EQUALS the LogFileName, or + in --solution mode matches `_*` or `*___*`. Never a substring of the line.""" + key = expr_norm(key) + literal = re.sub(r"([*?\[])", r"[\1]", key) + for a in args: + if not a or a.startswith("-") or a in WORD_BREAK: + continue + base = a.rsplit("/", 1)[-1] + if mode == "--solution": + if fnmatch.fnmatchcase(base, literal + "_*") or fnmatch.fnmatchcase(base, "*__" + literal + "_*"): + return True + elif base == key: + return True + return False + + +def heredoc_bodies(text, kind, block_of): + """{physical line number: the line its heredoc opened on}, for every line inside a heredoc body.""" + if kind not in ("shell", "workflow"): + return {} # a Make recipe line is a shell of its own; a compose command is folded into one line + lines = text.replace(CR + NL, NL).split(NL) + body, pending, cur = {}, [], None + for n, line in enumerate(lines, 1): + if cur is not None: + delim, dash, opened, blk = cur + if kind == "workflow" and block_of(n) != blk: + cur, pending = None, [] # the run block ended first; this line is read normally below + else: + # A workflow's `run: |` block is de-indented by YAML before bash sees it. + probe = line.strip() if kind == "workflow" else (line.lstrip(TAB) if dash else line) + if probe == delim: + cur = pending.pop(0) if pending else None + else: + body[n] = opened + continue + if (kind == "workflow" and block_of(n) is None) or is_comment(line): + continue + for m in HEREDOC_OP.finditer(masked(line)): + w = HEREDOC_WORD.match(line, m.end()) + if w: + pending.append((w.group(2), m.group(1) == "-", n, block_of(n))) + if pending and not line.rstrip().endswith(BS): + cur = pending.pop(0) + return body + + +def quiet_quit(cmd): + """The `exit`/`return` in this command that ends its shell with success (no status: the last + command's, which under set -e is 0; or a literal 0), or None.""" + for m in QUIT.finditer(masked(cmd)): + arg = QUIT_ARG.match(cmd, m.end()) + val = next((g for g in arg.groups() if g is not None), "") + if re.fullmatch(r"0*", val): + return (m.group(1) + " " + val).strip() + return None + + +def guard_defect(kind, text, raw): + """Why this guard line cannot fail its unit, or None when it can.""" + cmd, prefix = command_of(kind, text, raw) + if "-" in prefix: + return "make ignores its exit status (the `-` recipe prefix)" + if not GUARD_COMMAND.match(cmd): + return ("it is not a command of its own: the line must start with `bash .../" + GUARD_PATH + "` " + "(under `echo`, `if`, `!` and the like its failure never fails the lane)") + op = CONTROL.search(unquoted(cmd)) + if op: + return (f"it shares its line with `{op.group(0)}`, which lets the line succeed while the guard " + "fails; give the guard a line of its own") + return None + + +def errexit_state(entries, upto, block_of, block, initial): + """Whether `set -e` is in force at entry `upto`, reading the toggles before it in the same unit.""" + state = initial + for i in range(upto): + line_no, t, raw = entries[i] + if is_comment(raw) or block_of(line_no) != block: + continue + marks = [(m.start(), True) for m in SET_E.finditer(t)] + [(m.start(), False) for m in SET_NO_E.finditer(t)] + for _, on in sorted(marks): + state = on + return state + + +class Unit: + """One scanned file: its logical lines, what unit each line runs in, and its guard calls.""" + + def __init__(self, root, path): + self.name = rel(root, path) + self.kind = kind_of(self.name) + with open(path, encoding="utf-8", errors="replace") as fh: + self.text = fh.read() + self.found = variables(self.text, self.kind == "make") + self.entries = logical_lines(self.text, folded=self.kind == "compose") + if self.kind == "workflow": + blocks = run_blocks(self.text) + self.block_of = lambda n: blocks.get(n) + elif self.kind == "make": + targets, target = {}, None + for n, line in enumerate(self.text.replace(CR + NL, NL).split(NL), 1): + m = MAKE_TARGET.match(line) + if m and not line.startswith(TAB): + target = m.group(1) + targets[n] = target + self.block_of = lambda n: targets.get(n) + else: + self.block_of = lambda n: "file" + self.heredoc = heredoc_bodies(self.text, self.kind, self.block_of) + self.guards = [] + self.bringups = [] + for idx, (line_no, t, raw) in enumerate(self.entries): + if is_comment(raw): + continue + if GUARD_PATH in t: + self.guards.append(Guard(idx, t, self.block_of(line_no), self.defect(idx, t, raw), + guard_args(self.kind, t, raw))) + call = COMPOSE_CALL.search(t) + if self.kind != "compose" and call and COMPOSE_UP.search(call.group(1)): + for f in COMPOSE_FILE_ARG.findall(call.group(1)): + f = resolve(f, self.found) + self.bringups.append((f[2:] if f.startswith("./") else f, idx)) + + def defect(self, idx, t, raw): + line_no = self.entries[idx][0] + if line_no in self.heredoc: + return (f"it is text inside a heredoc body (opened at line {self.heredoc[line_no]}): data for " + "whatever reads the heredoc, not a command this unit runs") + why = guard_defect(self.kind, t, raw) + if why is not None: + return why + if self.kind == "shell" and not errexit_state(self.entries, idx, self.block_of, "file", False): + return "`set -e` is not in force where it runs, so the script carries on when it fails" + if self.kind == "workflow" and not errexit_state( + self.entries, idx, self.block_of, self.block_of(line_no), True): + return "a `set +e` earlier in its run block means the step carries on when it fails" + return None + + def where(self, idx): + return f"{self.name}:{self.entries[idx][0]}" + + def unit_label(self, block): + if self.kind == "make": + return f"Make target {block}" + if self.kind == "workflow": + return f"the run block at line {block}" if block is not None else "no run block" + return "the script" + + def quit_between(self, after, until, block): + """(entry, text) of an `exit`/`return` that ends the unit with success between two entries.""" + if self.kind == "make": + return None # each recipe line is a shell of its own: an `exit 0` line ends only itself + for i in range(after + 1, until): + line_no, t, raw = self.entries[i] + if is_comment(raw) or line_no in self.heredoc or self.block_of(line_no) != block: + continue + quit_ = quiet_quit(command_of(self.kind, t, raw)[0]) + if quit_: + return i, quit_ + return None + + def find_guard(self, after, key, mode): + """None when a guard that can fail follows entry `after` in its unit; else why not.""" + block = self.block_of(self.entries[after][0]) + naming = [g for g in self.guards if g.idx > after and names_trx(g.args, key, mode)] + same = [g for g in naming if block is not None and g.block == block] + if not same: + if naming: + return (f"is guarded only in another unit ({self.where(naming[0].idx)}, " + f"{self.unit_label(naming[0].block)}); a guard counts only in the unit that ran the " + f"test ({self.unit_label(block)})") + return f"is not followed by `bash {GUARD_PATH}` naming {key}" + why = None + for g in same: + if g.defect is not None: + why = why or f"is followed by a guard over {key} at {self.where(g.idx)} that cannot fail: {g.defect}" + continue + if mode and mode not in g.args: + why = why or f"needs its guard over {key} in {mode} mode ({self.where(g.idx)})" + continue + quit_ = self.quit_between(after, g.idx, block) + if quit_: + why = why or (f"can end before its guard over {key} at {self.where(g.idx)} runs: `{quit_[1]}` at " + f"{self.where(quit_[0])} ends {self.unit_label(block)} with success first") + continue + return None + return why + + +def scan(root, strict): + problems, pinned_total, wired, exempt_hits, fixture_hits = [], 0, set(), {}, {} + units = [Unit(root, p) for p in files_to_scan(root)] + bringups = [(u, f, idx) for u in units for f, idx in u.bringups] + for u in units: + file_key = next((p for p, _ in EXEMPT_FILES if fnmatch.fnmatch(u.name, p)), None) + fixture_key = next((p for p, _ in FIXTURE_TEXT if fnmatch.fnmatch(u.name, p)), None) + names_seen = {} + for idx, (line_no, t, raw) in enumerate(u.entries): + if is_comment(raw) or "dotnet test" not in t: + continue + for seg in t.split("dotnet test")[1:]: + fm = FILTER.search(seg) + if not fm: + continue + flt = resolve(next(g for g in fm.groups() if g is not None), u.found) + if not is_pinned(flt): + continue + if fixture_key is not None: + fixture_hits[fixture_key] = fixture_hits.get(fixture_key, 0) + 1 + continue + pinned_total += 1 + where = f"{u.name}:{line_no}" + if file_key is not None: + exempt_hits[("file", file_key)] = exempt_hits.get(("file", file_key), 0) + 1 + continue + if u.kind == "make": + target = u.block_of(line_no) + if any(tg == target for tg, _ in EXEMPT_MAKE_TARGETS): + exempt_hits[("make", target)] = exempt_hits.get(("make", target), 0) + 1 + continue + + proj = target_of(seg, u.found) + prefix = LOG_PREFIX.search(seg) + logname = LOG_NAME.search(seg) + if proj is None or "$" in proj: + problems.append(f"{where}: cannot resolve which project this pinned run targets ({proj!r}); " + "the unknown direction is not allowed to pass") + continue + if proj.endswith((".sln", ".slnf")): + if not prefix: + problems.append(f"{where}: solution-wide pinned run needs LogFilePrefix=

so each project " + "writes its own TRX (one LogFileName is overwritten project by project)") + continue + key, mode = prefix.group(1), "--solution" + else: + if not logname: + problems.append(f"{where}: pinned filter {flt!r} writes no TRX of its own; add " + '--logger "trx;LogFileName=.trx" and guard it') + continue + key, mode = logname.group(1), None + if key in names_seen: + problems.append(f"{where}: LogFileName={key} is also used at line {names_seen[key]}; " + "a shared name silently overwrites the earlier run's results") + names_seen[key] = line_no + if multi_target(root, proj) and not FRAMEWORK.search(seg): + problems.append(f"{where}: {proj} targets several frameworks and this run pins none; each " + "target overwrites the previous one's TRX. Pin -f, one invocation per framework") + + if u.kind == "compose": + # The TRX lands on the host of whoever brought the service up; the guard goes there. + sites = [(b, i) for b, f, i in bringups if f == u.name] + if not sites: + problems.append(f"{where}: pinned filter {flt!r} runs in a compose service that no " + f"workflow, script or Make target brings up with `docker compose -f " + f"{u.name} ... up`, so there is no place its guard could run") + continue + faults = [f"{where}: pinned filter {flt!r}, brought up at {b.where(i)}, {why}" + for b, i in sites for why in [b.find_guard(i, key, mode)] if why] + problems += faults + if not faults: + wired.add((u.name, key)) + continue + why = u.find_guard(idx, key, mode) + if why: + problems.append(f"{where}: pinned filter {flt!r} {why}") + continue + wired.add((u.name, key)) + + if strict: + if pinned_total < MINIMUM_PINNED: + problems.append(f"found {pinned_total} pinned invocations, expected at least {MINIMUM_PINNED}: the scan " + "has stopped matching, so a clean result would mean nothing. Fix the scan; do not " + "lower the floor") + for f, key in REQUIRED_WIRED: + if (f, key) not in wired: + problems.append(f"required lane {f} ({key}) was not found wired; the scan no longer reaches it") + for p, reason in EXEMPT_FILES: + if not exempt_hits.get(("file", p)): + problems.append(f"stale exemption {p} ({reason}): it matches no pinned invocation any more; remove it") + for tg, reason in EXEMPT_MAKE_TARGETS: + if not exempt_hits.get(("make", tg)): + problems.append(f"stale exemption Makefile:{tg} ({reason}): it matches no pinned invocation any more; " + "remove it") + for p, reason in FIXTURE_TEXT: + if not fixture_hits.get(p): + problems.append(f"stale fixture-text entry {p} ({reason}): it matches no `dotnet test` any more; " + "remove it") + return problems, pinned_total, len(wired), sum(exempt_hits.values()), sum(fixture_hits.values()) + + +def main(argv): + root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) + strict = True + if len(argv) == 3 and argv[1] == "--root": + root, strict = os.path.abspath(argv[2]), False + elif len(argv) != 1: + print("usage: verify-zero-test-guard-wiring.py [--root DIR]", file=sys.stderr) + return 2 + problems, pinned, wired, exempt, fixture = scan(root, strict) + for p in problems: + print(f"zero-test-guard-wiring: ERROR: {p}") + print(f"zero-test-guard-wiring: {pinned} pinned invocation(s): {wired} guarded, {exempt} exempt, " + f"{len(problems)} problem(s); {fixture} more in fixture text, not lanes") + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/scripts/ci/zero-test-guard.sh b/scripts/ci/zero-test-guard.sh new file mode 100755 index 000000000..42dfa8541 --- /dev/null +++ b/scripts/ci/zero-test-guard.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# Fail closed when a pinned `dotnet test --filter` ran nothing. +# +# WHY. VSTest exits 0 when a --filter matches no test: it prints "No test matches the given testcase +# filter", writes a TRX with total="0" executed="0", and the lane is green having asserted nothing. +# Measured on master at 876cc527 in the devtest container: security-gate Tier B selected ZERO tests +# (its API classes compile on net10.0 only and it ran net8.0), compat-gate Tier A's first step +# selected ZERO (its class moved to another project), and application-gate Tier C selected ZERO (same +# net8.0/net10.0 split). Each printed its own "PASS". scripts/cert-gate-zero-test-guard.sh closed this +# for cert-gate alone; this is the same fail-closed check for every other pinned filter, so a filter +# that goes stale turns its lane red instead of quiet. +# +# USAGE +# bash scripts/ci/zero-test-guard.sh [ ...] +# Each TRX is judged on its own, and every one of them must pass. Give each test invocation its +# own LogFileName: a multi-target invocation (no -f) that reuses one name OVERWRITES the first +# target's TRX with the second's ("WARNING: Overwriting results file"), so a target that +# matched nothing is invisible in the file that survives. Measured: the security-gate Tier B +# filter run without -f left a TRX saying 44 executed while its net8.0 leg had matched nothing. +# Pin -f per invocation, one TRX each. +# +# bash scripts/ci/zero-test-guard.sh --solution [ ...] +# For ONE invocation over a solution or solution filter, where most projects legitimately match +# nothing and each writes its own TRX (use LogFilePrefix, not LogFileName). Passes when at least +# one TRX executed a test; still fails on a missing or unreadable TRX, or when none executed. +# +# A TRX FAILS when +# - it does not exist: the run wrote nothing, or wrote it somewhere else. A count that cannot be +# read is not zero and it is not a pass, the same rule cert-gate-zero-test-guard.sh applies when +# it cannot derive its expected count; +# - its element, or the total/executed attributes on it, cannot be read; +# - executed == 0; +# - a RunInfo carries VSTest's own no-selection line ("No test matches the given testcase filter" +# or "No test is available"). Anchored on so a test that merely prints the phrase to +# stdout does not trip it. +# +# SKIPPED TESTS COUNT AS NOT RUN. In a TRX written by VSTest 18.x a skipped test is counted in +# `total` and NOT in `executed`, and `notExecuted` stays 0 - measured: a filter selecting exactly one +# [Fact(Skip = ...)] wrote total="1" executed="0" notExecuted="0". So skips are total - executed, and +# this guard counts executed only: +# - a selection that is ENTIRELY skipped fails. It asserted exactly as much as a selection of +# nothing, and "every test I pinned is skipped" is how a pinned lane goes quiet without its filter +# changing at all; +# - a selection that is PARTLY skipped passes and prints the skip count. Several pinned lanes select +# environment-gated tests that skip on a runner without Docker or a model; failing on any skip +# would make those lanes red by construction rather than by defect. +# This guard answers "did the filter select anything that ran", not "did everything it selected run". +# It is not a floor: a lane that shrinks from 40 tests to 1 still passes here. +# +# Honour the test step's own exit code by calling this AFTER it, under `set -e` (or in a later +# command of the same Make recipe): a failing test run exits before this is reached, and this can +# only ever add a failure, never mask one. +set -euo pipefail + +usage() { + echo "usage: zero-test-guard.sh [--solution] [ ...]" >&2 + exit 2 +} + +MODE="each" +if [[ "${1:-}" == "--solution" ]]; then + MODE="solution" + shift +fi +[[ $# -ge 1 ]] || usage + +FAILED=0 +ANY_EXECUTED=0 + +fail() { + echo "zero-test-guard: FAIL ${1}: ${2}" + FAILED=1 +} + +attr() { + # Prints the numeric value of attribute $1 in the element text $2, or nothing. + printf '%s' "$2" | sed -n "s/.*[[:space:]]${1}=\"\([0-9][0-9]*\)\".*/\1/p" +} + +for trx in "$@"; do + if [[ ! -f "${trx}" ]]; then + fail "${trx}" "no TRX was written. Nothing proves the filter selected a test." + continue + fi + + counters="$(grep -o ']*>' "${trx}" | head -n 1 || true)" + total="$(attr total "${counters}")" + executed="$(attr executed "${counters}")" + if [[ -z "${counters}" || -z "${total}" || -z "${executed}" ]]; then + fail "${trx}" "cannot read total/executed from its element; a count that cannot be read is not a pass." + continue + fi + skipped=$((total - executed)) + + no_selection=0 + if grep -qE '(No test matches the given testcase filter|No test is available)' "${trx}"; then + no_selection=1 + fi + + if [[ "${executed}" -gt 0 ]]; then + ANY_EXECUTED=1 + fi + + if [[ "${MODE}" == "solution" ]]; then + echo "zero-test-guard: ${trx}: executed=${executed} total=${total} skipped=${skipped}" + continue + fi + + if [[ "${no_selection}" -eq 1 ]]; then + fail "${trx}" "VSTest reports the filter selected nothing (executed=${executed}, total=${total})." + elif [[ "${executed}" -eq 0 && "${total}" -gt 0 ]]; then + fail "${trx}" "all ${total} selected test(s) were skipped, so the lane asserted nothing." + elif [[ "${executed}" -eq 0 ]]; then + fail "${trx}" "the filter selected no test (executed=0, total=0)." + else + echo "zero-test-guard: ok ${trx}: executed=${executed} total=${total} skipped=${skipped}" + fi +done + +if [[ "${MODE}" == "solution" && "${ANY_EXECUTED}" -eq 0 && "${FAILED}" -eq 0 ]]; then + echo "zero-test-guard: FAIL: no project in the solution-wide run executed a test." + FAILED=1 +fi + +if [[ "${FAILED}" -ne 0 ]]; then + echo "zero-test-guard: a pinned filter that selects nothing passes green and proves nothing; fix the" + echo " filter, its target framework or its project - do not delete the guard." + exit 1 +fi diff --git a/scripts/compat-gate-tier-a.sh b/scripts/compat-gate-tier-a.sh index 53c434cca..e4b550a3a 100755 --- a/scripts/compat-gate-tier-a.sh +++ b/scripts/compat-gate-tier-a.sh @@ -5,22 +5,40 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT" INFRA="src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj" +FLEET_TESTS="commercial/tests/Ashlar.Commercial.Tests.Fleet/Ashlar.Commercial.Tests.Fleet.csproj" +TRX_DIR="test-results/compat-gate-tier-a" +rm -rf "$TRX_DIR" echo "== Compat Tier A: mesh checkpoint migration ==" -dotnet build "$INFRA" -v minimal -dotnet test "$INFRA" -f net8.0 --no-build \ +# This step ran against Tests.Infrastructure and selected ZERO tests from 2026-06-08 until this change. +# The filter was written on 2026-05-21 (bec2a6ed) when MeshTaskExecutionServiceTests lived in +# src/Nexo.Tests.Infrastructure/Tests/Fleet; the commercial extraction (678d0718, #151) moved it to +# commercial/tests/Nexo.Commercial.Tests.Fleet, and the step kept pointing at the old project. The +# filter string is unchanged - it is the project that was wrong. Measured in the dev container at +# 876cc527: Tests.Infrastructure selects 0, Commercial.Tests.Fleet selects 1 +# (MigrateForCheckpointAsync_clears_assignment_and_sets_checkpoint). The GapCoverage class's own +# MigrateForCheckpointAsync_* facts (3 more) postdate this filter and are deliberately not added here. +dotnet build "$FLEET_TESTS" -f net8.0 -v minimal +dotnet test "$FLEET_TESTS" -f net8.0 --no-build \ --filter "FullyQualifiedName~MeshTaskExecutionServiceTests.MigrateForCheckpointAsync" \ + --logger "trx;LogFileName=mesh-checkpoint-migration.trx" --results-directory "$TRX_DIR" \ --blame-hang-timeout 60s --blame-hang-dump-type none +bash scripts/ci/zero-test-guard.sh "$TRX_DIR/mesh-checkpoint-migration.trx" echo "== Compat Tier A: LiteDB persistence registration ==" +dotnet build "$INFRA" -v minimal dotnet test "$INFRA" -f net8.0 --no-build \ --filter "FullyQualifiedName~PipelineServiceCollectionExtensionsTests.AddPipelineCompositionLayer_WithLiteDbPersistence" \ + --logger "trx;LogFileName=litedb-persistence-registration.trx" --results-directory "$TRX_DIR" \ --blame-hang-timeout 60s --blame-hang-dump-type none +bash scripts/ci/zero-test-guard.sh "$TRX_DIR/litedb-persistence-registration.trx" echo "== Compat Tier A: composition registry validation ==" dotnet test "$INFRA" -f net8.0 --no-build \ --filter "FullyQualifiedName~CompositionRegistryValidationTests" \ + --logger "trx;LogFileName=composition-registry-validation.trx" --results-directory "$TRX_DIR" \ --blame-hang-timeout 60s --blame-hang-dump-type none +bash scripts/ci/zero-test-guard.sh "$TRX_DIR/composition-registry-validation.trx" echo "" echo "compat-gate-tier-a: PASS" diff --git a/scripts/compat-gate-tier-c.sh b/scripts/compat-gate-tier-c.sh index e4cc6d769..760e2e22b 100755 --- a/scripts/compat-gate-tier-c.sh +++ b/scripts/compat-gate-tier-c.sh @@ -6,17 +6,23 @@ cd "$ROOT" INFRA="src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj" CLI="application/src/Ashlar.CLI/Ashlar.CLI.csproj" +TRX_DIR="test-results/compat-gate-tier-c" +rm -rf "$TRX_DIR" echo "== Compat Tier C: configuration override tests ==" dotnet build "$INFRA" -v minimal dotnet test "$INFRA" -f net8.0 --no-build \ --filter "FullyQualifiedName~PipelineServiceCollectionExtensionsTests.AddPipelineCompositionLayer_WithConfiguration" \ + --logger "trx;LogFileName=configuration-override.trx" --results-directory "$TRX_DIR" \ --blame-hang-timeout 60s --blame-hang-dump-type none +bash scripts/ci/zero-test-guard.sh "$TRX_DIR/configuration-override.trx" echo "== Compat Tier C: hosting profile resolution ==" dotnet test "$INFRA" -f net8.0 --no-build \ --filter "FullyQualifiedName~KernelPhaseResolutionTests" \ + --logger "trx;LogFileName=hosting-profile-resolution.trx" --results-directory "$TRX_DIR" \ --blame-hang-timeout 180s --blame-hang-dump-type none +bash scripts/ci/zero-test-guard.sh "$TRX_DIR/hosting-profile-resolution.trx" echo "== Compat Tier C: doctor smoke ==" dotnet build "$CLI" -v minimal >/dev/null diff --git a/scripts/composition-mesh-gate-tier-a.sh b/scripts/composition-mesh-gate-tier-a.sh index 2554c7af6..0500b08b8 100755 --- a/scripts/composition-mesh-gate-tier-a.sh +++ b/scripts/composition-mesh-gate-tier-a.sh @@ -5,12 +5,16 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT" INFRA="src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj" +TRX_DIR="test-results/composition-mesh-gate-tier-a" +rm -rf "$TRX_DIR" echo "== Composition Tier A: pipeline composition tests ==" dotnet build "$INFRA" -f net8.0 -v minimal ASHLAR_ALLOW_MOCK=1 dotnet test "$INFRA" -f net8.0 --no-build \ --filter "FullyQualifiedName~Ashlar.Tests.Infrastructure.Tests.Pipelines" \ + --logger "trx;LogFileName=pipeline-composition.trx" --results-directory "$TRX_DIR" \ --blame-hang-timeout 180s --blame-hang-dump-type none +bash scripts/ci/zero-test-guard.sh "$TRX_DIR/pipeline-composition.trx" echo "" echo "composition-mesh-gate-tier-a: PASS" diff --git a/scripts/composition-mesh-gate-tier-b.sh b/scripts/composition-mesh-gate-tier-b.sh index c6b1e5900..e55ad96a0 100755 --- a/scripts/composition-mesh-gate-tier-b.sh +++ b/scripts/composition-mesh-gate-tier-b.sh @@ -5,12 +5,16 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT" CLI="application/src/Ashlar.Tests.CLI/Ashlar.Tests.CLI.csproj" +TRX_DIR="test-results/composition-mesh-gate-tier-b" +rm -rf "$TRX_DIR" echo "== Composition Tier B: open mesh CLI unit suites ==" dotnet build "$CLI" -v minimal ASHLAR_ALLOW_MOCK=1 dotnet test "$CLI" -f net10.0 --no-build \ --filter "FullyQualifiedName~UnitTestBridgeTests&(DisplayName~PipelineCommand|DisplayName~MeshCommand|DisplayName~OptimizeAgentCluster)" \ + --logger "trx;LogFileName=mesh-cli.trx" --results-directory "$TRX_DIR" \ --blame-hang-timeout 180s --blame-hang-dump-type none +bash scripts/ci/zero-test-guard.sh "$TRX_DIR/mesh-cli.trx" echo "" echo "composition-mesh-gate-tier-b: PASS" diff --git a/scripts/dr-gate-tier-b.sh b/scripts/dr-gate-tier-b.sh index 3b8043205..c5eb1092b 100755 --- a/scripts/dr-gate-tier-b.sh +++ b/scripts/dr-gate-tier-b.sh @@ -5,12 +5,16 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT" INFRA="src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj" +TRX_DIR="test-results/dr-gate-tier-b" +rm -rf "$TRX_DIR" echo "== DR Tier B: LiteDB user knowledge log store ==" dotnet build "$INFRA" -v minimal dotnet test "$INFRA" -f net8.0 --no-build \ --filter "FullyQualifiedName~LiteDbUserKnowledgeLogStoreTests" \ + --logger "trx;LogFileName=knowledge-log-store.trx" --results-directory "$TRX_DIR" \ --blame-hang-timeout 60s --blame-hang-dump-type none +bash scripts/ci/zero-test-guard.sh "$TRX_DIR/knowledge-log-store.trx" REPORT_DIR=".ashlar/dr-gate" mkdir -p "$REPORT_DIR" diff --git a/scripts/kernel-gate-tier-b.sh b/scripts/kernel-gate-tier-b.sh index bdf9cbf29..3fd81cf8a 100755 --- a/scripts/kernel-gate-tier-b.sh +++ b/scripts/kernel-gate-tier-b.sh @@ -14,6 +14,8 @@ trap 'rm -rf "$TMP_ROOT"' EXIT TEMPLATE_PATH="$TMP_ROOT/pipeline_gate_demo.json" RESUME_DB="$TMP_ROOT/ashlar_pipeline_gate_resume.db" +TRX_DIR="test-results/kernel-gate-tier-b" +rm -rf "$TRX_DIR" echo "== Tier B: build checks ==" dotnet build src/Ashlar.Core.Application/Ashlar.Core.Application.csproj -f netstandard2.0 -v minimal @@ -24,7 +26,9 @@ echo "== Tier B: pipeline lifecycle tests (net8) ==" ASHLAR_ALLOW_MOCK=1 dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj -f net8.0 \ --filter "FullyQualifiedName~PipelineTemplateValidatorTests|FullyQualifiedName~PipelineLifecycleE2ETests" \ --blame-hang-timeout 180s --blame-hang-dump-type none \ + --logger "trx;LogFileName=pipeline-lifecycle.trx" --results-directory "$TRX_DIR" \ --logger "console;verbosity=minimal" +bash scripts/ci/zero-test-guard.sh "$TRX_DIR/pipeline-lifecycle.trx" cat > "$TEMPLATE_PATH" <<'JSON' { diff --git a/scripts/kernel-gate-tier-c.sh b/scripts/kernel-gate-tier-c.sh index 32d7e972d..5b8e2d46b 100755 --- a/scripts/kernel-gate-tier-c.sh +++ b/scripts/kernel-gate-tier-c.sh @@ -8,6 +8,8 @@ cd "$ROOT" INFRA="src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj" TRANSPORT="src/Ashlar.Tests.Transport/Ashlar.Tests.Transport.csproj" +TRX_DIR="test-results/kernel-gate-tier-c" +rm -rf "$TRX_DIR" echo "== Tier C: ProdStyle Infrastructure (net8, FluentAssertions-safe filter) ==" make test-prod-style @@ -16,14 +18,18 @@ echo "== Tier C: workflow executor integration ==" dotnet build "$INFRA" -v minimal ASHLAR_ALLOW_MOCK=1 dotnet test "$INFRA" -f net8.0 --no-build \ --filter "FullyQualifiedName~WorkflowExecutorIntegrationTests" \ + --logger "trx;LogFileName=workflow-executor.trx" --results-directory "$TRX_DIR" \ --blame-hang-timeout 120s --blame-hang-dump-type none +bash scripts/ci/zero-test-guard.sh "$TRX_DIR/workflow-executor.trx" echo "== Tier C: gRPC transport ProdStyle ==" if [ -f "$TRANSPORT" ]; then dotnet build "$TRANSPORT" -v minimal dotnet test "$TRANSPORT" -f net8.0 --no-build \ --filter "Category=ProdStyle" \ + --logger "trx;LogFileName=transport-prodstyle.trx" --results-directory "$TRX_DIR" \ --blame-hang-timeout 120s --blame-hang-dump-type none + bash scripts/ci/zero-test-guard.sh "$TRX_DIR/transport-prodstyle.trx" else echo "Skip: $TRANSPORT not found" fi @@ -31,7 +37,9 @@ fi echo "== Tier C: air-gapped profile smoke (in-process) ==" ASHLAR_ALLOW_MOCK=1 dotnet test "$INFRA" -f net8.0 --no-build \ --filter "FullyQualifiedName~AirGapped" \ + --logger "trx;LogFileName=air-gapped-profile.trx" --results-directory "$TRX_DIR" \ --blame-hang-timeout 180s --blame-hang-dump-type none +bash scripts/ci/zero-test-guard.sh "$TRX_DIR/air-gapped-profile.trx" if [ "${KERNEL_GATE_MESH_E2E:-0}" = "1" ] && [ -f ".env.mesh-lab" ]; then echo "== Tier C: mesh virtual lab E2E (compose up + verify + down) ==" diff --git a/scripts/kernel-gate-tier-e.sh b/scripts/kernel-gate-tier-e.sh index 1beac214c..151cab88f 100755 --- a/scripts/kernel-gate-tier-e.sh +++ b/scripts/kernel-gate-tier-e.sh @@ -9,18 +9,24 @@ cd "$ROOT" INFRA="src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj" ORCH="src/Ashlar.Tests.Orchestration/Ashlar.Tests.Orchestration.csproj" +TRX_DIR="test-results/kernel-gate-tier-e" +rm -rf "$TRX_DIR" echo "== Tier E: OpenTelemetry registration ==" dotnet build "$INFRA" -v minimal dotnet test "$INFRA" -f net8.0 --no-build \ --filter "FullyQualifiedName~OpenTelemetryTests" \ + --logger "trx;LogFileName=opentelemetry.trx" --results-directory "$TRX_DIR" \ --blame-hang-timeout 180s --blame-hang-dump-type none +bash scripts/ci/zero-test-guard.sh "$TRX_DIR/opentelemetry.trx" echo "== Tier E: orchestration performance-scoped tests ==" dotnet build "$ORCH" -v minimal dotnet test "$ORCH" -f net8.0 --no-build \ --filter "FullyQualifiedName~Ashlar.Tests.Orchestration.Performance" \ + --logger "trx;LogFileName=orchestration-performance.trx" --results-directory "$TRX_DIR" \ --blame-hang-timeout 120s --blame-hang-dump-type none +bash scripts/ci/zero-test-guard.sh "$TRX_DIR/orchestration-performance.trx" if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then echo "== Tier E: production-shaped Compose dry run (portal) ==" diff --git a/scripts/perf-gate-tier-a.sh b/scripts/perf-gate-tier-a.sh index 3e383e071..b58dab831 100755 --- a/scripts/perf-gate-tier-a.sh +++ b/scripts/perf-gate-tier-a.sh @@ -6,6 +6,8 @@ cd "$ROOT" REPORT_DIR=".ashlar/perf" mkdir -p "$REPORT_DIR" +# A stale TRX from an earlier local run would satisfy the guard below without this run writing one. +rm -f "$REPORT_DIR/perf-orchestration.trx" "$REPORT_DIR/perf-background-agents.trx" ORCH="src/Ashlar.Tests.Orchestration/Ashlar.Tests.Orchestration.csproj" BG="src/Ashlar.Tests.BackgroundAgents/Ashlar.Tests.BackgroundAgents.csproj" @@ -17,6 +19,7 @@ dotnet test "$ORCH" -f net8.0 --no-build \ --logger "trx;LogFileName=perf-orchestration.trx" \ --results-directory "$REPORT_DIR" \ --blame-hang-timeout 120s --blame-hang-dump-type none +bash scripts/ci/zero-test-guard.sh "$REPORT_DIR/perf-orchestration.trx" echo "== Perf Tier A: background agent performance tests ==" dotnet build "$BG" -v minimal @@ -25,6 +28,7 @@ dotnet test "$BG" -f net8.0 --no-build \ --logger "trx;LogFileName=perf-background-agents.trx" \ --results-directory "$REPORT_DIR" \ --blame-hang-timeout 120s --blame-hang-dump-type none +bash scripts/ci/zero-test-guard.sh "$REPORT_DIR/perf-background-agents.trx" echo "" echo "perf-gate-tier-a: PASS" diff --git a/scripts/release/readiness-verdict-for-sha.py b/scripts/release/readiness-verdict-for-sha.py new file mode 100644 index 000000000..391186210 --- /dev/null +++ b/scripts/release/readiness-verdict-for-sha.py @@ -0,0 +1,415 @@ +#!/usr/bin/env python3 +"""Decide what the Full Platform Readiness Gate proved about one commit. + +Called by scripts/release/readiness-verdict-for-sha.sh, which does the GitHub fetching; this file +only reads JSON and decides, so every branch of the decision can be driven offline from a fixture +(tests/scripts/readiness-verdict-for-sha.test.sh). Not meant to be run by hand. + + plan

read /check-runs.ndjson and print what else the shell must fetch + evaluate dir decide from what the shell fetched into + evaluate fixture decide from one evidence document (offline; never evidence itself) + +An evidence document is {"check_runs": [...], "annotations": {"": [...]}, +"workflow_runs": [...]}, each object shaped as the GitHub REST API returns it. workflow_runs holds +the Actions run behind each check run, plus any run of the readiness workflow for the SHA that has +no check run yet (one still in progress). + +Exit 0 VERIFIED, 1 REFUSED, 2 UNDETERMINED, and 3 for a VERIFIED read from an offline fixture: a +fixture is never evidence, so it never exits 0. The last line of output always names which, and why. +""" +import io +import json +import os +import re +import sys + +for _stream in (sys.stdout, sys.stderr): + try: + # Windows python writes CRLF by default, and a trailing CR on the verdict line makes it + # compare unequal to the same text on Linux. + _stream.reconfigure(newline="\n") + except AttributeError: + pass + +PREFIX = "readiness-verdict-for-sha" +CHECK_NAME = "Readiness summary" +ANNOTATION_TITLE = "Readiness verdict" +WORKFLOW_FILE = "full-platform-readiness-gate.yml" +WORKFLOW_PATH = ".github/workflows/" + WORKFLOW_FILE +ACTIONS_APP = "github-actions" +# Written by the `Summarize results` step of the readiness-summary job. Change both together. +MESSAGE = re.compile( + r"^verdict=(verified|partial|not-verified|failed) lanes_ran=([0-9]+) lanes_skipped=([0-9]+)$") +JOB_URL = re.compile(r"/actions/runs/([0-9]+)/job/[0-9]+") + +EXIT = {"VERIFIED": 0, "REFUSED": 1, "UNDETERMINED": 2} +# A fixture is never evidence, so a VERIFIED reading of one must not exit 0: otherwise +# `READINESS_VERDICT_FIXTURE= bash readiness-verdict-for-sha.sh $SHA && git tag ...` tags on a +# test file, and so would any gate built on this exit code. REFUSED and UNDETERMINED already exit +# non-zero and keep their codes in fixture mode, so a test can still tell them apart. +FIXTURE_VERIFIED_EXIT = 3 + + +def evidence_route(sha): + short = sha[:12] + return [ + "", + "To get evidence for exactly this commit, run the gate on a ref that points at it, wait for", + "the run to complete, and run this script again:", + "", + " git push origin %s:refs/heads/readiness-evidence/%s" % (sha, short), + " gh workflow run %s --ref readiness-evidence/%s" % (WORKFLOW_FILE, short), + " # ...when that run has completed:", + " bash scripts/release/readiness-verdict-for-sha.sh %s" % sha, + " git push origin --delete readiness-evidence/%s" % short, + "", + "A workflow_dispatch runs every lane group unconditionally. But `gh workflow run --ref`", + "takes a branch or tag, not a SHA, and the run evaluates whatever the ref points at when it", + "starts. Dispatching on master is evidence for this SHA only if master still points at it", + "then, and a master dispatch shares the push runs' concurrency group, so the next merge", + "that starts a readiness run cancels it. A throwaway branch pinned to the SHA has neither", + "problem. Never use a v* tag for this: that is the release trigger. This script reads only", + "check runs attached to this exact SHA, so a dispatch that resolved to some other commit", + "can never count here.", + ] + + +def rerun_caution(): + # Every piece of advice that says `gh run rerun` carries this. A re-run keeps the original + # run's ref (GitHub documents that it reuses GITHUB_SHA and GITHUB_REF), and the workflow's + # concurrency group is keyed on that ref with cancel-in-progress: true. + return [ + "", + "A re-run is not free on master. It keeps the original run's ref, so re-running a push,", + "schedule or dispatch run of master puts it in master's concurrency group", + "(full-platform-readiness-refs/heads/master, cancel-in-progress: true): starting it cancels", + "any master readiness run in flight - whose own summary then records 'failed' - and the next", + "merge that touches a readiness path cancels the re-run. Re-run only when this shows no run", + "queued or in progress, and hold readiness-path merges until the re-run has completed:", + " gh run list --workflow %s --branch master --limit 5" % WORKFLOW_FILE, + ] + + +class Outcome(Exception): + def __init__(self, status, reason, explanation): + Exception.__init__(self, reason) + self.status = status + self.reason = reason + self.explanation = explanation + + +def refuse(reason, *lines): + raise Outcome("REFUSED", reason, list(lines)) + + +def undetermined(reason, *lines): + raise Outcome("UNDETERMINED", reason, list(lines)) + + +def read_ndjson(path): + out = [] + with io.open(path, encoding="utf-8") as fh: + for number, line in enumerate(fh, 1): + line = line.strip() + if not line: + continue + try: + out.append(json.loads(line)) + except ValueError as exc: + undetermined("malformed-evidence", "%s:%d is not JSON: %s" % (path, number, exc)) + return out + + +def job_run_id(check_run): + match = JOB_URL.search(str(check_run.get("details_url") or "")) + return match.group(1) if match else None + + +def plan(directory): + """Print, one per line, what the shell still has to fetch: annotations and Actions runs.""" + runs = read_ndjson(os.path.join(directory, "check-runs.ndjson")) + seen = set() + for check_run in runs: + if not isinstance(check_run, dict) or check_run.get("name") != CHECK_NAME: + continue + cid = str(check_run.get("id", "")) + if cid.isdigit(): + print("annotations " + cid) + rid = job_run_id(check_run) + if rid and rid not in seen: + seen.add(rid) + print("run " + rid) + + +def load_directory(directory): + check_runs = read_ndjson(os.path.join(directory, "check-runs.ndjson")) + listing = os.path.join(directory, "workflow-runs.ndjson") + workflow_runs = read_ndjson(listing) if os.path.isfile(listing) else [] + annotations = {} + for check_run in check_runs: + if not isinstance(check_run, dict): + continue + cid = str(check_run.get("id", "")) + path = os.path.join(directory, "annotations-%s.ndjson" % cid) + if cid.isdigit() and os.path.isfile(path): + annotations[cid] = read_ndjson(path) + rid = job_run_id(check_run) + path = os.path.join(directory, "run-%s.json" % rid) + if rid and os.path.isfile(path): + with io.open(path, encoding="utf-8") as fh: + try: + # Fetched by id, so it is authoritative; it goes last and wins over the listing. + workflow_runs.append(json.load(fh)) + except ValueError as exc: + undetermined("malformed-evidence", "%s is not JSON: %s" % (path, exc)) + return {"check_runs": check_runs, "annotations": annotations, "workflow_runs": workflow_runs} + + +def load_fixture(path): + try: + with io.open(path, encoding="utf-8") as fh: + return json.load(fh) + except (IOError, OSError) as exc: + undetermined("fixture-unreadable", "READINESS_VERDICT_FIXTURE=%s cannot be read: %s" % (path, exc)) + except ValueError as exc: + undetermined("malformed-evidence", "READINESS_VERDICT_FIXTURE=%s is not JSON: %s" % (path, exc)) + + +def read_attempt(check_run, annotations): + """One check run -> (state, verdict, detail). state is ok|pending|missing|malformed|inconsistent.""" + status = check_run.get("status") + conclusion = check_run.get("conclusion") + if status != "completed": + return "pending", None, "status %s" % status + entries = annotations.get(str(check_run.get("id"))) + titled = [a for a in (entries or []) if isinstance(a, dict) and a.get("title") == ANNOTATION_TITLE] + if not titled: + return "missing", None, "completed/%s, no '%s' annotation" % (conclusion, ANNOTATION_TITLE) + if len(titled) > 1: + return "malformed", None, "%d annotations titled '%s'" % (len(titled), ANNOTATION_TITLE) + message = str(titled[0].get("message") or "").strip() + match = MESSAGE.match(message) + if not match: + return "malformed", None, "annotation message %r does not parse" % message + verdict, ran, skipped = match.group(1), int(match.group(2)), int(match.group(3)) + shape_ok = { + "verified": ran > 0 and skipped == 0, + "partial": ran > 0 and skipped > 0, + "not-verified": ran == 0, + "failed": True, + }[verdict] + expected_conclusion = "failure" if verdict == "failed" else "success" + if not shape_ok or conclusion != expected_conclusion: + return "inconsistent", verdict, "%s: check conclusion is %s" % (message, conclusion) + return "ok", verdict, "completed/%s, %s" % (conclusion, message) + + +def attempt_order(check_run): + # A re-run of the same workflow run adds a check run; the one that started last is the attempt + # branch protection shows, and it is the one that counts. The id breaks a tie. + cid = check_run.get("id") + return (str(check_run.get("started_at") or ""), cid if isinstance(cid, int) else -1) + + +def evaluate(sha, doc, lines): + if not isinstance(doc, dict) or not isinstance(doc.get("check_runs"), list): + undetermined("malformed-evidence", "the evidence has no check_runs list.") + annotations = doc.get("annotations") or {} + workflow_runs = {} + for run in doc.get("workflow_runs") or []: + if isinstance(run, dict) and run.get("id") is not None: + workflow_runs[str(run["id"])] = run + + candidates = [c for c in doc["check_runs"] if isinstance(c, dict) and c.get("name") == CHECK_NAME] + if not candidates: + mine = [r for r in workflow_runs.values() + if r.get("path") == WORKFLOW_PATH and r.get("head_sha") == sha] + active = sorted(str(r["id"]) for r in mine if r.get("status") != "completed") + if active: + refuse("in-progress", + "Actions run(s) %s of %s are still running for this commit and have not" % (", ".join(active), WORKFLOW_FILE), + "reached the '%s' job yet. Wait for them to complete and run this script again." % CHECK_NAME) + if mine: + refuse("no-readiness-run", *([ + "%d completed run(s) of %s exist for this commit (%s), but none has a" % (len(mine), WORKFLOW_FILE, ", ".join(sorted(str(r["id"]) for r in mine))), + "'%s' check run, so none recorded a verdict. Re-run it: gh run rerun " % CHECK_NAME, + ] + rerun_caution())) + refuse("no-readiness-run", *([ + "No '%s' check run exists for this commit, so nothing says any platform was checked." % CHECK_NAME, + "", + "That is expected for some commits and it is NOT a pass: on push to master the readiness", + "gate is PATH-FILTERED, so a commit whose diff touches no readiness path (a docs-only", + "release commit, for example) gets no run at all. Pull request runs attach to the PR's", + "head commit, which after a squash merge is not the commit on master. (If the commit was", + "pushed moments ago, its run may not be listed yet: wait a minute and retry first.)", + ] + evidence_route(sha))) + + # Provenance, before anything is read. A check run of this name from some other app or workflow + # is refused rather than ignored: ignoring it is the one mistake that could let it count. + by_run = {} + for check_run in candidates: + cid = check_run.get("id") + if check_run.get("head_sha") != sha: + refuse("sha-mismatch", "check run %s is attached to %s, not %s." % (cid, check_run.get("head_sha"), sha)) + slug = (check_run.get("app") or {}).get("slug") + if slug != ACTIONS_APP: + refuse("foreign-check-run", + "check run %s named '%s' was created by app '%s', not GitHub Actions." % (cid, CHECK_NAME, slug)) + rid = job_run_id(check_run) + run = workflow_runs.get(rid) if rid else None + if not isinstance(run, dict): + refuse("foreign-check-run", + "check run %s (details_url %s) cannot be tied to an Actions run of %s." + % (cid, check_run.get("details_url"), WORKFLOW_PATH)) + if run.get("path") != WORKFLOW_PATH: + refuse("foreign-check-run", + "check run %s comes from workflow '%s', not %s." % (cid, run.get("path"), WORKFLOW_PATH)) + if run.get("head_sha") != sha: + refuse("sha-mismatch", "Actions run %s evaluated %s, not %s." % (rid, run.get("head_sha"), sha)) + by_run.setdefault(rid, []).append(check_run) + + readings = [] # (rid, event, state, verdict, check_run) + for rid in sorted(by_run, key=int): + attempts = sorted(by_run[rid], key=attempt_order) + latest = attempts[-1] + run = workflow_runs[rid] + state, verdict, detail = read_attempt(latest, annotations) + if run.get("status") != "completed": + # A re-run in progress has not created its new summary check run yet, so the latest + # check run is still the previous attempt's. The run's own status is what says so. + state, verdict = "pending", None + detail += "; but Actions run status is %s (a newer attempt is running)" % run.get("status") + lines.append(" Actions run %s (%s): check run %s %s" + % (rid, run.get("event", "?"), latest.get("id"), detail)) + lines.append(" %s" % (latest.get("details_url") or run.get("html_url") or "")) + for older in attempts[:-1]: + _, _, older_detail = read_attempt(older, annotations) + lines.append(" superseded by the later attempt: check run %s %s" % (older.get("id"), older_detail)) + readings.append((rid, run.get("event", "?"), state, verdict, latest)) + + # A run of the workflow with no summary check run at all yet: still running, still able to fail. + for rid in sorted(workflow_runs, key=lambda k: int(k) if k.isdigit() else -1): + run = workflow_runs[rid] + if (rid not in by_run and run.get("path") == WORKFLOW_PATH and run.get("head_sha") == sha + and run.get("status") != "completed"): + lines.append(" Actions run %s (%s): status %s, no '%s' check run yet" + % (rid, run.get("event", "?"), run.get("status"), CHECK_NAME)) + readings.append((rid, run.get("event", "?"), "pending", None, None)) + + def runs_in(*states): + return [r for r in readings if r[2] in states] + + pending = runs_in("pending") + if pending: + refuse("in-progress", *[ + "%d run(s) above have not completed, and an unfinished run could still report 'failed'." % len(pending), + "Wait for them to complete and run this script again.", + ]) + missing = runs_in("missing") + if missing: + refuse("missing-annotation", *([ + "%d run(s) above carry no '%s' annotation, so what they proved cannot be read." % (len(missing), ANNOTATION_TITLE), + "Either the workflow at this commit predates the annotation (any verdict it computed is", + "then only in the run's step summary, which the Checks API does not expose), or the", + "summary job ended before emitting it (cancelled, or an earlier step failed). A green tick", + "without the annotation does not distinguish 'every platform passed' from 'none ran'.", + "If the run was cancelled or broke early, re-run it (gh run rerun ): the later", + "attempt supersedes this one. A commit that predates the annotation cannot be certified", + "by this script at all; read its verdict from the run summary and say so in the release.", + ] + rerun_caution())) + broken = runs_in("malformed", "inconsistent") + if broken: + refuse("malformed-annotation", *[ + "The '%s' annotation on %d run(s) above does not parse, or contradicts its own check" % (ANNOTATION_TITLE, len(broken)), + "conclusion. The workflow and this script disagree about the format; fix them together", + "(%s, job readiness-summary) rather than reading around it." % WORKFLOW_PATH, + ]) + + verdicts = sorted(set(r[3] for r in readings)) + if "failed" in verdicts: + failed_ids = ", ".join(r[0] for r in readings if r[3] == "failed") + advice = [ + "Open the failed run (Actions run %s). If a lane was cancelled because a newer push to" % failed_ids, + "the same ref superseded it, or failed on a known flake, re-run THAT run with", + "gh run rerun ; its latest attempt then supersedes the failure. Otherwise the", + "commit is not ready.", + "", + "A dispatch on a throwaway branch does NOT clear this. The failed run stays attached to", + "this commit, and a verified dispatch beside it is conflicting-verdicts. Only a later", + "attempt of the failed run itself supersedes its failure.", + ] + rerun_caution() + if len(verdicts) > 1: + refuse("conflicting-verdicts", *([ + "Runs of this commit disagree: %s. A failure is not outweighed by a pass in" % ", ".join(verdicts), + "another run, and this script will not choose between them.", + ] + advice)) + refuse("failed", *(["A lane group failed or was cancelled on this commit."] + advice)) + + # A pull_request run's check is attached to the PR head, but it checked out and tested + # refs/pull/N/merge - this commit merged into its base, not this commit. Today such a run can + # never be `verified` (it skips docker-all-images); if the workflow ever changes that, it still + # must not certify this SHA. + verified_here = [r for r in readings if r[3] == "verified" and r[1] != "pull_request"] + if verified_here: + weaker = sorted(set(r[3] if r[1] != "pull_request" else r[3] + " (pull_request)" + for r in readings if r not in verified_here)) + note = [] + if weaker: + note = ["Other runs reported %s: passes that exercised fewer lanes or another tree," % ", ".join(weaker), + "which do not contradict a run in which every lane group ran and passed here."] + raise Outcome("VERIFIED", "verified", [ + "Every lane group of the readiness gate ran and passed on this exact commit.", + ] + note) + + if "verified" in verdicts: + refuse("merge-commit-only", *([ + "The only verified run is a pull_request run. Its check is attached to this commit, but", + "it tested this commit merged into its base branch, not this commit alone.", + ] + evidence_route(sha))) + + if "partial" in verdicts: + refuse("partial", *([ + "Some lane groups ran and passed and others were skipped, so this is not platform", + "verification of the commit, and it is refused. On push, schedule and workflow_dispatch", + "every lane group runs; a partial verdict comes from a pull_request or merge_group run,", + "which never builds the production Docker images (docker-all-images).", + ] + evidence_route(sha))) + + refuse("not-verified", *([ + "Every lane group was skipped: the check is green because nothing in the diff reached a", + "readiness path, not because any platform was exercised.", + ] + evidence_route(sha))) + + +def main(argv): + if len(argv) >= 2 and argv[0] == "plan": + plan(argv[1]) + return 0 + if len(argv) != 4 or argv[0] != "evaluate" or argv[2] not in ("dir", "fixture"): + sys.stderr.write(__doc__) + return 2 + sha, source, where = argv[1], argv[2], argv[3] + lines = [] + if source == "fixture": + lines.append("%s: OFFLINE FIXTURE MODE - evidence read from %s, not from GitHub." % (PREFIX, where)) + lines.append("%s: '%s' check runs for %s" % (PREFIX, CHECK_NAME, sha)) + try: + doc = load_fixture(where) if source == "fixture" else load_directory(where) + evaluate(sha, doc, lines) + outcome = Outcome("UNDETERMINED", "no-decision", ["internal error: no decision was reached."]) + except Outcome as reached: + outcome = reached + for line in lines + [""] + outcome.explanation + [""]: + print(line) + suffix = " [offline fixture - not evidence]" if source == "fixture" else "" + if outcome.status == "VERIFIED": + print("%s: %s VERIFIED%s" % (PREFIX, sha, suffix)) + if source == "fixture": + return FIXTURE_VERIFIED_EXIT + else: + print("%s: %s %s (%s)%s" % (PREFIX, sha, outcome.status, outcome.reason, suffix)) + return EXIT[outcome.status] + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/release/readiness-verdict-for-sha.sh b/scripts/release/readiness-verdict-for-sha.sh new file mode 100755 index 000000000..c170f14f2 --- /dev/null +++ b/scripts/release/readiness-verdict-for-sha.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# Print the verdict the Full Platform Readiness Gate recorded for ONE commit, and exit 0 only if it +# is `verified`. +# +# WHY. `Readiness summary` is a required check that must pass when its heavy lanes are skipped, so +# its green tick cannot tell "every platform passed" from "no platform ran". The job computes a +# verdict that can (verified | partial | not-verified | failed), but job outputs are readable only +# inside the run and the step summary is not exposed by the Checks API. The job therefore also +# emits the verdict as a check-run annotation titled "Readiness verdict", and this script reads +# that annotation for a SHA. A release sign-off cites this script's output, not the tick. +# +# Usage: bash scripts/release/readiness-verdict-for-sha.sh +# +# Exit: 0 VERIFIED every lane group ran and passed on this exact commit, read from GitHub. +# Nothing else exits 0. +# 1 REFUSED the recorded evidence does not support a sign-off; the last line says why +# (no-readiness-run, missing-annotation, conflicting-verdicts, failed, +# partial, not-verified, merge-commit-only, in-progress, +# malformed-annotation, foreign-check-run, sha-mismatch). +# 2 UNDETERMINED this script could not read the evidence (usage, no python, gh missing or +# unauthenticated, an API error, an unreadable fixture). +# 3 FIXTURE READINESS_VERDICT_FIXTURE was set and the fixture reads as VERIFIED. A +# fixture is never evidence, so this is never 0, and +# `READINESS_VERDICT_FIXTURE=... bash $SHA && git tag` does +# not tag. A fixture that reads REFUSED or UNDETERMINED keeps 1 or 2. +# +# `partial` is REFUSED, with no override. On push, schedule and workflow_dispatch every lane group +# runs, so only a pull_request or merge_group run can be partial - and those never build the +# production images. A pull_request run also tests refs/pull/N/merge rather than the SHA its check +# is attached to, so even a `verified` from one is refused (merge-commit-only). +# +# Several runs of one SHA (a push run and a dispatch, say) are each read at their LATEST attempt, as +# branch protection reads a re-run. Any `failed` among them refuses, and if another run passed that +# is reported as conflicting-verdicts; partial or not-verified beside a verified run is weaker +# evidence, not a contradiction. +# +# Repository: whatever gh resolves for `{owner}/{repo}` - the git remote of the current directory, +# or GH_REPO=owner/repo. Read-only: this script only issues GET requests. +# +# Offline: READINESS_VERDICT_FIXTURE= reads one evidence document instead of calling +# GitHub (shape in scripts/release/readiness-verdict-for-sha.py). Output then says so on its first +# and last lines, and a verified reading exits 3 rather than 0, because a fixture is never evidence. +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EVALUATOR="${HERE}/readiness-verdict-for-sha.py" +PREFIX="readiness-verdict-for-sha" + +undetermined() { + local reason="$1"; shift + local line + for line in "$@"; do echo "${line}"; done + echo + echo "${PREFIX}: ${SHA:-} UNDETERMINED (${reason})" + exit 2 +} + +SHA="${1:-}" +if [[ $# -ne 1 || ! "${SHA}" =~ ^[0-9a-f]{40}$ ]]; then + SHA="" + undetermined usage \ + "usage: bash scripts/release/readiness-verdict-for-sha.sh " \ + "Pass the commit you are about to tag, in full and lower case: git rev-parse HEAD" \ + "A branch name or short SHA is refused on purpose - the API would resolve it to whatever it" \ + "points at when asked, which is not necessarily the commit being released." +fi + +# PROBE the interpreter, do not merely locate it: on Windows `python3` can be an App Execution +# Alias stub that exists, is executable, and exits 49 without running anything. +PY_BIN="" +for candidate in python3 python py; do + if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c 'print(1)' >/dev/null 2>&1; then + PY_BIN="${candidate}" + break + fi +done +[[ -n "${PY_BIN}" ]] || undetermined no-python "no working python on PATH (tried python3, python, py)." +[[ -f "${EVALUATOR}" ]] || undetermined no-evaluator "missing ${EVALUATOR}" + +# The fixture path goes to python as an ARGUMENT, never through the environment: MSYS rewrites a +# POSIX path argument for a native Windows python, but not an environment variable. +if [[ -n "${READINESS_VERDICT_FIXTURE:-}" ]]; then + "${PY_BIN}" "${EVALUATOR}" evaluate "${SHA}" fixture "${READINESS_VERDICT_FIXTURE}" + exit $? +fi + +command -v gh >/dev/null 2>&1 || undetermined gh-missing \ + "gh (the GitHub CLI) is not on PATH. Install it and run: gh auth login" + +WORK="$(mktemp -d)" +trap 'rm -rf "${WORK}"' EXIT + +if ! gh auth status >"${WORK}/auth.txt" 2>&1; then + undetermined gh-unauthenticated \ + "gh is not authenticated, so the check runs for ${SHA} cannot be read. gh auth status said:" \ + "$(sed 's/^/ /' "${WORK}/auth.txt")" \ + "Run: gh auth login (read access to the repository is enough). Refusing to guess." +fi + +# fetch +fetch() { + local endpoint="$1" filter="$2" out="$3" + local -a args=(api "${endpoint}") + [[ -n "${filter}" ]] && args=(api --paginate "${endpoint}" --jq "${filter}") + if ! gh "${args[@]}" >"${out}" 2>"${WORK}/err.txt"; then + undetermined api-error \ + "GET ${endpoint} failed. gh said:" \ + "$(sed 's/^/ /' "${WORK}/err.txt")" \ + "A commit GitHub has never seen (not pushed) answers 422 here." + fi +} + +# filter=all, not the default filter=latest: every attempt comes back, so the evaluator can say +# which attempt it read and which it superseded instead of trusting an unseen server-side choice. +fetch "repos/{owner}/{repo}/commits/${SHA}/check-runs?check_name=Readiness%20summary&filter=all&per_page=100" \ + '.check_runs[]' "${WORK}/check-runs.ndjson" +# Only used to explain an absence: a run still in progress has no summary check run yet, and that +# must not be reported as "the path filter skipped this commit". Provenance is checked against the +# per-run fetch below, never against this listing (measured: it can transiently omit a run). +fetch "repos/{owner}/{repo}/actions/workflows/full-platform-readiness-gate.yml/runs?head_sha=${SHA}&per_page=100" \ + '.workflow_runs[]' "${WORK}/workflow-runs.ndjson" + +PLAN="$("${PY_BIN}" "${EVALUATOR}" plan "${WORK}")" \ + || undetermined malformed-evidence "could not read the check runs GitHub returned for ${SHA}." +while read -r kind id; do + id="${id%$'\r'}" + [[ "${id}" =~ ^[0-9]+$ ]] || continue + case "${kind}" in + annotations) fetch "repos/{owner}/{repo}/check-runs/${id}/annotations?per_page=100" '.[]' "${WORK}/annotations-${id}.ndjson" ;; + run) fetch "repos/{owner}/{repo}/actions/runs/${id}" '' "${WORK}/run-${id}.json" ;; + esac +done <<< "${PLAN}" + +"${PY_BIN}" "${EVALUATOR}" evaluate "${SHA}" dir "${WORK}" +exit $? diff --git a/scripts/run-cert-gate.sh b/scripts/run-cert-gate.sh index b5d013bb1..50d583558 100755 --- a/scripts/run-cert-gate.sh +++ b/scripts/run-cert-gate.sh @@ -36,10 +36,13 @@ dotnet test src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj \ TEST_EXIT=$? set -e +# Both guards run whatever the tests returned: dotnet test exits 0 when the filter matches nothing +# (zero-test guard) and when tests are skipped (skip guard), so its exit status alone proves neither. bash scripts/cert-gate-zero-test-guard.sh "${TRX}" +bash scripts/cert-gate-skip-guard.sh "${TRX}" -if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then - bash scripts/cert-gate-summary.sh "${TRX}" -fi +# The job summary is written by the workflow's own `if: always()` step +# (.github/workflows/cert-gate.yml), which also covers a failed run. It used to be written here as +# well, which put the same table in the CI job summary twice. exit "${TEST_EXIT}" diff --git a/scripts/security-gate-tier-a.sh b/scripts/security-gate-tier-a.sh index ce63f4697..16dd5cdb5 100755 --- a/scripts/security-gate-tier-a.sh +++ b/scripts/security-gate-tier-a.sh @@ -5,12 +5,16 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT" INFRA="src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj" +TRX_DIR="test-results/security-gate-tier-a" +rm -rf "$TRX_DIR" echo "== Security Tier A: trust core (policy packs + peer trust + audit) ==" dotnet build "$INFRA" -f net8.0 -v minimal ASHLAR_ALLOW_MOCK=1 dotnet test "$INFRA" -f net8.0 --no-build \ --filter "FullyQualifiedName~Ashlar.Tests.Infrastructure.Tests.Trust|FullyQualifiedName~AshlarPeerBrickExecutorTrustTests" \ + --logger "trx;LogFileName=trust-core.trx" --results-directory "$TRX_DIR" \ --blame-hang-timeout 180s --blame-hang-dump-type none +bash scripts/ci/zero-test-guard.sh "$TRX_DIR/trust-core.trx" echo "" echo "security-gate-tier-a: PASS" diff --git a/scripts/security-gate-tier-b.sh b/scripts/security-gate-tier-b.sh index f00e6d946..4faa9b727 100755 --- a/scripts/security-gate-tier-b.sh +++ b/scripts/security-gate-tier-b.sh @@ -5,12 +5,40 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT" INFRA="src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj" +TRX_DIR="test-results/security-gate-tier-b" +rm -rf "$TRX_DIR" echo "== Security Tier B: API security middleware ==" -dotnet build "$INFRA" -f net8.0 -v minimal -ASHLAR_ALLOW_MOCK=1 dotnet test "$INFRA" -f net8.0 --no-build \ - --filter "FullyQualifiedName~AshlarApiKeyAuthMiddlewareTests|FullyQualifiedName~MeshSecurityMiddlewareTests|FullyQualifiedName~AshlarApiOpenInternetReadinessTests|FullyQualifiedName~SecurityAdvisoryEndpointTests|FullyQualifiedName~SecurityAnalysisRuleTests" \ +# This lane selected ZERO tests and printed PASS from 2026-08-17 until this change. It ran -f net8.0, +# and e6b5579f made Tests/API compile on net10.0 only (Ashlar.API ships on net10.0 alone), so the four +# API classes below stopped existing in the assembly it tested. Measured in the dev container at +# 876cc527: net8.0 selects 0, net10.0 selects 44 (16 + 5 + 18 + 5). +dotnet build "$INFRA" -f net10.0 -v minimal +ASHLAR_ALLOW_MOCK=1 dotnet test "$INFRA" -f net10.0 --no-build \ + --filter "FullyQualifiedName~AshlarApiKeyAuthMiddlewareTests|FullyQualifiedName~MeshSecurityMiddlewareTests|FullyQualifiedName~AshlarApiOpenInternetReadinessTests|FullyQualifiedName~SecurityAdvisoryEndpointTests" \ + --logger "trx;LogFileName=api-security.trx" --results-directory "$TRX_DIR" \ --blame-hang-timeout 180s --blame-hang-dump-type none +bash scripts/ci/zero-test-guard.sh "$TRX_DIR/api-security.trx" + +echo "== Security Tier B: security analysis rule (UnitTestBase suite) ==" +# The filter above used to end in |FullyQualifiedName~SecurityAnalysisRuleTests, which was never +# selectable: SecurityAnalysisRuleTests is a `UnitTestBase` suite, not an xUnit class, so VSTest has no +# test by that name on either framework. The only runner that reaches it is the UnitTestBridgeTests +# theory, one row per suite, selected by DisplayName - the pattern security-gate-tier-c.sh uses. +# +# It is a SEPARATE invocation with its own TRX on purpose. Folded into the filter above, this one row +# exists on net8.0 too, so reverting the framework left the lane at 1 executed test and the guard +# green while all 44 API tests vanished again - measured by mutation. A zero guard can only protect a +# selection it sees on its own. +# +# 720s because the bridge row carries Timeout = TestTimeouts.HostTouching (480s), and a window under +# 1.5x the widest per-test deadline turns a stall into a killed host that names nothing +# (docs/Testing.md). +ASHLAR_ALLOW_MOCK=1 dotnet test "$INFRA" -f net10.0 --no-build \ + --filter "FullyQualifiedName~UnitTestBridgeTests&DisplayName~SecurityAnalysisRuleTests" \ + --logger "trx;LogFileName=security-analysis-rule.trx" --results-directory "$TRX_DIR" \ + --blame-hang-timeout 720s --blame-hang-dump-type none +bash scripts/ci/zero-test-guard.sh "$TRX_DIR/security-analysis-rule.trx" echo "" echo "security-gate-tier-b: PASS" diff --git a/scripts/security-gate-tier-c.sh b/scripts/security-gate-tier-c.sh index 79fe83fbd..5aad0ccc4 100755 --- a/scripts/security-gate-tier-c.sh +++ b/scripts/security-gate-tier-c.sh @@ -6,6 +6,8 @@ cd "$ROOT" CLI="application/src/Ashlar.CLI/Ashlar.CLI.csproj" CLI_TESTS="application/src/Ashlar.Tests.CLI/Ashlar.Tests.CLI.csproj" +TRX_DIR="test-results/security-gate-tier-c" +rm -rf "$TRX_DIR" echo "== Security Tier C: package-admission and untrusted-output suites ==" dotnet build "$CLI_TESTS" -v minimal @@ -32,7 +34,9 @@ dotnet build "$CLI_TESTS" -v minimal # workflow's `paths:` select the CLI directories these suites cover. ASHLAR_ALLOW_MOCK=1 dotnet test "$CLI_TESTS" -f net10.0 --no-build \ --filter "FullyQualifiedName~SafePackageReadTests|FullyQualifiedName~PkgCommandTests|FullyQualifiedName~UntrustedTextTests|FullyQualifiedName~MeshLanPartyTests|(FullyQualifiedName~UnitTestBridgeTests&DisplayName~TrustCommandTests)" \ + --logger "trx;LogFileName=trust-cli.trx" --results-directory "$TRX_DIR" \ --blame-hang-timeout 120s --blame-hang-dump-type none +bash scripts/ci/zero-test-guard.sh "$TRX_DIR/trust-cli.trx" echo "== Security Tier C: trust boundary + dashboard JSON smoke ==" dotnet build "$CLI" -v minimal diff --git a/scripts/security-gate-tier-e.sh b/scripts/security-gate-tier-e.sh index 70fb5bc69..b03c6646d 100755 --- a/scripts/security-gate-tier-e.sh +++ b/scripts/security-gate-tier-e.sh @@ -6,12 +6,16 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT" INFRA="src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj" +TRX_DIR="test-results/security-gate-tier-e" +rm -rf "$TRX_DIR" echo "== Security Tier E: air-gapped + safety in-process tests ==" dotnet build "$INFRA" -f net8.0 -v minimal ASHLAR_ALLOW_MOCK=1 dotnet test "$INFRA" -f net8.0 --no-build \ --filter "FullyQualifiedName~AirGapped|FullyQualifiedName~Ashlar.Tests.Infrastructure.Tests.Safety" \ + --logger "trx;LogFileName=air-gapped-safety.trx" --results-directory "$TRX_DIR" \ --blame-hang-timeout 180s --blame-hang-dump-type none +bash scripts/ci/zero-test-guard.sh "$TRX_DIR/air-gapped-safety.trx" if [ "${SECURITY_GATE_AIRGAPPED_CONTAINER:-0}" = "1" ]; then if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then diff --git a/scripts/ship-gate-tier-a.sh b/scripts/ship-gate-tier-a.sh index e22e9a8f8..d6067683d 100755 --- a/scripts/ship-gate-tier-a.sh +++ b/scripts/ship-gate-tier-a.sh @@ -7,6 +7,8 @@ cd "$ROOT" CLI="application/src/Ashlar.CLI/Ashlar.CLI.csproj" INFRA_TESTS="src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj" +TRX_DIR="test-results/ship-gate-tier-a" +rm -rf "$TRX_DIR" if ! command -v python3 >/dev/null 2>&1; then echo "python3 is required for JSON assertions in ship-gate-tier-a" >&2 @@ -22,7 +24,9 @@ echo "== Ship Tier A: host DI smoke ==" dotnet build "$INFRA_TESTS" -f net8.0 -v minimal ASHLAR_ALLOW_MOCK=1 dotnet test "$INFRA_TESTS" -f net8.0 --no-build \ --filter "FullyQualifiedName~HostingE2ESmokeTests.AddAshlar_RegistersObservationPipeline_ByDefault|FullyQualifiedName~PipelineServiceCollectionExtensionsTests.AddAshlar_RegistersPipelineCompositionLayerByDefault" \ + --logger "trx;LogFileName=host-di-smoke.trx" --results-directory "$TRX_DIR" \ --blame-hang-timeout 180s --blame-hang-dump-type none +bash scripts/ci/zero-test-guard.sh "$TRX_DIR/host-di-smoke.trx" TMP="$(mktemp -d)" export TMP diff --git a/scripts/ship-gate-tier-b.sh b/scripts/ship-gate-tier-b.sh index cbec6c208..f824286c2 100755 --- a/scripts/ship-gate-tier-b.sh +++ b/scripts/ship-gate-tier-b.sh @@ -7,6 +7,8 @@ cd "$ROOT" CLI="application/src/Ashlar.CLI/Ashlar.CLI.csproj" INFRA="src/Ashlar.Tests.Infrastructure/Ashlar.Tests.Infrastructure.csproj" +TRX_DIR="test-results/ship-gate-tier-b" +rm -rf "$TRX_DIR" echo "== Ship Tier B: build CLI + infrastructure tests ==" dotnet build "$CLI" -v minimal @@ -18,7 +20,9 @@ make test-prod-style echo "== Ship Tier B: framework smoke ==" ASHLAR_ALLOW_MOCK=1 dotnet test "$INFRA" -f net8.0 --no-build \ --filter "FullyQualifiedName~BaseFrameworkSmokeTests" \ + --logger "trx;LogFileName=framework-smoke.trx" --results-directory "$TRX_DIR" \ --blame-hang-timeout 60s --blame-hang-dump-type none +bash scripts/ci/zero-test-guard.sh "$TRX_DIR/framework-smoke.trx" echo "== Ship Tier B: doctor --json ==" set +e diff --git a/src/Ashlar.Infrastructure/Validation/Adapters/ValidationServiceAdapter.cs b/src/Ashlar.Infrastructure/Validation/Adapters/ValidationServiceAdapter.cs index 8897cb1fe..67f9f08bd 100644 --- a/src/Ashlar.Infrastructure/Validation/Adapters/ValidationServiceAdapter.cs +++ b/src/Ashlar.Infrastructure/Validation/Adapters/ValidationServiceAdapter.cs @@ -212,18 +212,24 @@ public async Task ValidateAsync( } else { - switch (ClassifyCompletedRun(trxFile, recordedResults, run.Output)) + // The zero-execution rule is judged per project and on EXECUTED tests, not + // on the NoTestsSelected arm alone: a project whose every test is skipped + // records rows, classifies as ResultsRecorded, and would slip past a check + // keyed on the empty-selection arm. Invalid evidence already fails the + // project and is reported once, as itself. + var evidence = ClassifyCompletedRun(trxFile, recordedResults, run.Output); + var error = evidence == CompletedRunEvidence.InvalidEvidence + ? InvalidEvidenceError(testProject.Name) + : ZeroExecutionError(testProject.Name, filter, executed.Count, recordedResults - executed.Count); + if (error is not null) { - case CompletedRunEvidence.NoTestsSelected: - emptyProjects.Add(testProject.Name); - _logger.LogInformation("{Project}: no tests selected (0 run)", testProject.Name); - break; - case CompletedRunEvidence.InvalidEvidence: - var error = $"{testProject.Name}: test results are missing, unreadable or inconsistent; " - + "the run cannot be reported as passing."; - evidenceErrors.Add(error); - _logger.LogWarning("{Error}", error); - break; + evidenceErrors.Add(error); + _logger.LogWarning("{Error}", error); + } + else if (evidence == CompletedRunEvidence.NoTestsSelected) + { + emptyProjects.Add(testProject.Name); + _logger.LogInformation("{Project}: no tests selected (0 run)", testProject.Name); } } } @@ -244,7 +250,10 @@ public async Task ValidateAsync( } } - // Pass if no tests failed (even if no tests were run) + // Pass only when no test failed and every project produced usable evidence. A project + // that executed nothing in an unfiltered sweep is an evidence error (see + // ZeroExecutionError), so "no tests were run" no longer passes on its own; an empty + // selection under a caller's filter still does. var passed = totalTestsFailed == 0 && evidenceErrors.Count == 0; progress?.Report(new ProgressReport @@ -255,16 +264,12 @@ public async Task ValidateAsync( TotalSteps = totalProjects }); - var skippedSuffix = totalTestsSkipped > 0 ? $", {totalTestsSkipped} skipped" : string.Empty; - var evidenceSuffix = evidenceErrors.Count > 0 ? "; " + string.Join("; ", evidenceErrors) : string.Empty; - var emptySuffix = emptyProjects.Count > 0 ? "; no tests selected: " + string.Join(", ", emptyProjects) : string.Empty; return new ValidationResult { Passed = passed, - Message = (passed - ? $"Validation passed ({totalTestsPassed}/{totalTestsRun} tests{skippedSuffix})" - : $"Validation failed ({totalTestsFailed}/{totalTestsRun} tests failed{skippedSuffix})") - + evidenceSuffix + emptySuffix, + Message = DescribeOutcome( + passed, totalTestsRun, totalTestsPassed, totalTestsFailed, totalTestsSkipped, + evidenceErrors, emptyProjects), TestsRun = totalTestsRun, TestsPassed = totalTestsPassed, TestsFailed = totalTestsFailed, @@ -400,6 +405,83 @@ bool MatchesSummary(IReadOnlyList countedRows) } } + /// + /// Whether the caller narrowed the sweep. A blank filter is no filter: this is the same test + /// uses to decide whether to append the caller's + /// clause, so --filter "" cannot run an unfiltered sweep with the zero-execution rule + /// switched off. + /// + internal static bool CallerNarrowedTheSweep(string? filter) => !string.IsNullOrWhiteSpace(filter); + + /// The evidence error for a project whose result evidence cannot be trusted. + internal static string InvalidEvidenceError(string projectName) => + $"{projectName}: test results are missing, unreadable or inconsistent; " + + "the run cannot be reported as passing."; + + /// + /// The evidence error for a discovered test project that executed no test in a sweep the + /// caller did not narrow, or null when the caller passed a filter or the project executed at + /// least one test. Skips beside an executed test are fine - a documented skip is not a red + /// test - so only a project whose EXECUTED count is zero fails, whether it selected nothing or + /// every test it selected was skipped. + /// + /// + /// Policy, chosen conservatively: validate always excludes Category=Stress and + /// Category=DockerOptional (), so a project whose + /// every test carries one of those traits legitimately selects nothing, and it still fails + /// here. A sweep that reports "passed" for a project it ran nothing in is the lane-not-a-receipt + /// shape this repository keeps finding. A future all-Stress or all-DockerOptional test project + /// must therefore carry at least one test outside those categories (a fast structural or smoke + /// fact) so the sweep executes something in it. The same holds for a project whose every test + /// is SKIPPED wherever the sweep runs it: an integration suite made only of + /// [OptInFact] tests whose switch is unset, one made only of [NotOnCiFact] tests + /// under CI=true, or a suite skipped wholesale on one platform. It fails on the lane + /// where it executes nothing, and it must carry at least one test that runs there without the + /// dependency (a structural or wiring fact). There is deliberately no opt-out marker: an + /// opt-out is how a project goes quiet. A caller who narrows the sweep with a filter has chosen + /// the selection, so an empty selection stays a pass there and is reported as such. The rule is + /// the same whichever way the run was classified as empty: a zero-row TRX, or no TRX under the + /// project beside the console's "no test matches" line. + /// + internal static string? ZeroExecutionError(string projectName, string? filter, int executed, int skipped) + { + if (CallerNarrowedTheSweep(filter) || executed > 0) + return null; + + var why = skipped switch + { + 0 => "it selected none, and validate always excludes Category=Stress and Category=DockerOptional", + 1 => "its only selected test was skipped", + _ => $"all {skipped} of its selected tests were skipped", + }; + return $"{projectName}: executed no tests in an unfiltered sweep ({why}); " + + "a discovered test project that runs nothing is not evidence of passing."; + } + + /// + /// The one-line result validate reports. When nothing failed but a project produced no usable + /// evidence, the headline says so instead of "0/N tests failed", which reads as a + /// contradiction; the per-project reasons follow it in every case. + /// + internal static string DescribeOutcome( + bool passed, int testsRun, int testsPassed, int testsFailed, int testsSkipped, + IReadOnlyCollection evidenceErrors, IReadOnlyCollection emptyProjects) + { + var skippedSuffix = testsSkipped > 0 ? $", {testsSkipped} skipped" : string.Empty; + string headline; + if (passed) + headline = $"Validation passed ({testsPassed}/{testsRun} tests{skippedSuffix})"; + else if (testsFailed > 0 || evidenceErrors.Count == 0) + headline = $"Validation failed ({testsFailed}/{testsRun} tests failed{skippedSuffix})"; + else + headline = $"Validation failed ({testsPassed}/{testsRun} tests passed{skippedSuffix}, but " + + $"{evidenceErrors.Count} {(evidenceErrors.Count == 1 ? "project" : "projects")} produced no passing evidence)"; + + var evidenceSuffix = evidenceErrors.Count > 0 ? "; " + string.Join("; ", evidenceErrors) : string.Empty; + var emptySuffix = emptyProjects.Count > 0 ? "; no tests selected: " + string.Join(", ", emptyProjects) : string.Empty; + return headline + evidenceSuffix + emptySuffix; + } + private static readonly System.Text.RegularExpressions.Regex PlaceholderToken = new("__[A-Za-z0-9]+__", System.Text.RegularExpressions.RegexOptions.Compiled); @@ -742,7 +824,7 @@ internal static ProcessStartInfo CreateDotnetTestStartInfo( { ValidateFilterGrouping(filter); const string exclusions = "Category!=DockerOptional&Category!=Stress"; - var effectiveFilter = string.IsNullOrWhiteSpace(filter) ? exclusions : $"({exclusions})&({filter})"; + var effectiveFilter = CallerNarrowedTheSweep(filter) ? $"({exclusions})&({filter})" : exclusions; var startInfo = CreateDotnetStartInfo(csprojPath); startInfo.ArgumentList.Add("test"); startInfo.ArgumentList.Add(csprojPath); diff --git a/src/Ashlar.Tests.Infrastructure/Tests/Certification/CertifierReferenceSetTests.cs b/src/Ashlar.Tests.Infrastructure/Tests/Certification/CertifierReferenceSetTests.cs index 82b683c4f..d246842b3 100644 --- a/src/Ashlar.Tests.Infrastructure/Tests/Certification/CertifierReferenceSetTests.cs +++ b/src/Ashlar.Tests.Infrastructure/Tests/Certification/CertifierReferenceSetTests.cs @@ -38,10 +38,12 @@ namespace Ashlar.Tests.Infrastructure.Tests.Certification; /// is advisory — and is overdetermined here anyway: this class uses /// Ashlar.Core.Application.Certification.Ports and Ashlar.Infrastructure.Certification, /// which Ashlar.Analyzers.Tests does not reference, so it could not live beside the #605 -/// precedent whose design it copies. The remaining asymmetry is routing, not execution: cert-gate is -/// required and unfiltered, whereas readiness runs Ashlar.Analyzers.Tests through its native -/// ci verify -> validate sweep but lists no analyzer glob in either path list, so an -/// analyzer-only PR skips the heavy lanes. The design transfers; the placement must not. +/// precedent whose design it copies. The remaining asymmetry is +/// routing, not execution: cert-gate is required and runs on every pull request, whereas readiness +/// runs Ashlar.Analyzers.Tests through its native ci verify -> validate sweep only on a +/// diff matching its path lists - which, since 2026-09-30, include both src/Ashlar.Analyzers/** +/// and src/Ashlar.Analyzers.Tests/** - and Readiness summary counts a skipped lane as a +/// pass. The design transfers; the placement must not. /// /// Corrected 2026-09-13. This paragraph previously read "ci/test-ownership.tsv line 47 /// records as UNOWNED — no gate runs it". That premise was false: readiness runs 34737292726 and diff --git a/src/Ashlar.Tests.Infrastructure/Tests/Certification/ReadinessClosureConventionTests.cs b/src/Ashlar.Tests.Infrastructure/Tests/Certification/ReadinessClosureConventionTests.cs new file mode 100644 index 000000000..e9d90110d --- /dev/null +++ b/src/Ashlar.Tests.Infrastructure/Tests/Certification/ReadinessClosureConventionTests.cs @@ -0,0 +1,923 @@ +using System.Text.RegularExpressions; +using System.Xml.Linq; +using Ashlar.Core.Application.Paths; +using Ashlar.Infrastructure.Validation.Adapters; +using FluentAssertions; +using Xunit; +using Xunit.Abstractions; +using YamlDotNet.Core; +using YamlDotNet.RepresentationModel; + +namespace Ashlar.Tests.Infrastructure.Tests.Certification; + +/// +/// The readiness trigger must cover everything the suites it runs are built from. +/// +/// What was wrong. The native readiness lanes run ashlar ci verify, which runs +/// ashlar validate, which builds and runs every test project its recursive sweep discovers +/// (ValidationServiceAdapter.IsDiscoverableTestProject). Those lanes are diff-conditional: they +/// run only when a PR touches a glob in READINESS_PATHS. On 876cc527 the ProjectReference +/// closure of the discovered suites held 70 projects and 37 of their directories matched no glob, so +/// editing src/Ashlar.Mcp.Server (built into six suites) skipped every lane that runs them +/// while editing their tests re-ran them. #684 fixed that for the kernel suite alone. +/// +/// What this proves. Over the checked-out tree: (1) every project directory in the +/// transitive ProjectReference closure of every discovered suite is covered by a glob in BOTH +/// readiness lists - where a closure project's references are read from the project file AND from +/// every shared MSBuild file it is evaluated from (the implicit Directory.Build.props, +/// Directory.Build.targets and Directory.Packages.props above it, and any file reached +/// by an explicit <Import>), because a reference declared there is a real edge of every +/// project that reads it; (2) every implicit per-directory build input MSBuild or the compiler reads +/// for a closure project (Directory.Build.props/.targets/.rsp, +/// Directory.Packages.props, .editorconfig, .globalconfig, nuget.config, +/// global.json, in the project's directory or any ancestor) is covered; (3) every path +/// OUTSIDE the project's own directory that a closure project or one of those shared MSBuild files +/// names is covered, where a path is named by the Include of an item of ANY type except the identity +/// types (a package, framework, assembly or namespace name and the like) and a None item that is not +/// copied, by any item's HintPath, or by an <Import>. A failure names the +/// uncovered path, what names it, the list, and a suite that needs it. +/// +/// Decisions, stated so they are not rediscovered. Discovery calls the product +/// predicate itself rather than a copy, so the two cannot drift; it is handed repo-relative paths +/// under a virtual root because the predicate also reads the ABSOLUTE directory for the substring +/// "test", and CI's workspace (/home/runner/work/Ashlar/Ashlar and its macOS and Windows +/// equivalents) contains none - a checkout under a directory named *test* would otherwise +/// discover every project. References are read with an XML parser, so attribute order is irrelevant +/// (a regex on <ProjectReference\s+Include= misses +/// <ProjectReference OutputItemType="Analyzer" Include=...>). Analyzer-only +/// references (OutputItemType="Analyzer", ReferenceOutputAssembly="false") are +/// followed like any other: the analyzer runs inside the referencing project's compile, so changing +/// it changes whether the suite builds - the reason #684 routed Ashlar.Analyzers. +/// Conditional references are followed too: this lens does not evaluate MSBuild conditions, +/// and over-approximating costs one glob, where under-approximating costs a skipped lane. For the +/// same reason a reference in a shared MSBuild file becomes an edge of every closure project +/// below it, whatever $(MSBuildProjectName) condition guards it, and every ancestor +/// Directory.Build.* counts as read even where MSBuild would stop at the nearest one; a +/// shared file's reference to one of its own importers is dropped, since MSBuild refuses that cycle +/// and only a condition can make such a file build. Item types are a denylist of identity +/// types, not an allowlist of path types: a path-bearing type nobody listed (Protobuf, +/// EditorConfigFiles, a code-generation input) is scanned, and a computed path on one fails. +/// A bare file name in a shared file names a file in each importer's own directory, which that +/// project's glob covers; a path under $(NuGetPackageRoot) is in the package cache, outside +/// the repository. +/// +/// What this does NOT prove. That the lanes pass, or that a suite selects any test - +/// readiness counts a skipped lane as a pass and validate passes a project that selects zero +/// tests. Runtime file reads (a test opening ci/*.json through FindRepoRoot) are not +/// build inputs and are not traced, and neither is a project built by an <MSBuild> +/// task inside a target. Files read through an MSBuild property function are not traced either: +/// $([System.IO.File]::ReadAllText(...)) is how the root Directory.Build.props and +/// Directory.Build.targets read VERSION, which is in both lists by hand, not because +/// this proves it. Nor is a path in item metadata other than HintPath (a Protobuf +/// ProtoRoot or AdditionalImportDirs), or a file a compiled input reaches on its own (a +/// .proto import). Anything ci verify builds OTHER than the discovered suites is out of +/// scope. Unsupported shapes fail rather than being approximated: a $(Property) path, a +/// wildcard ProjectReference or Import, and a ProjectReference or item path in a shared MSBuild +/// file that is not anchored by $(MSBuildThisFileDirectory) (MSBuild resolves those against +/// each importing project, so they name no single file). +/// +public sealed class ReadinessClosureConventionTests +{ + private const string Readiness = ".github/workflows/full-platform-readiness-gate.yml"; + private const string KernelSuite = "src/Ashlar.Tests.Kernel/Ashlar.Tests.Kernel.csproj"; + private const string KernelDependency = "src/Ashlar.Manifest/Ashlar.Manifest.csproj"; + + // The two kinds of shared MSBuild file the reference walk must read: the implicit one every src/ + // project in the closure is evaluated from, and the tree's one explicit (from the + // Tests.Infrastructure project, a discovered suite). + private const string ImplicitSharedFile = "src/Directory.Build.props"; + private const string ExplicitSharedFile = "src/Ashlar.Tests.Infrastructure/CopyAssemblies.targets"; + + // Non-vacuity floors. STATED here and lowered only by a visible diff: a count regenerated from the + // tree would move down with the tree and could never fire. Measured on 876cc527 (2026-09-30): + // validate discovers 22 suites whose closure is 70 projects, with 10 implicit inputs and 6 + // cross-directory inputs. The floors sit below those numbers so routine reference pruning does not + // trip a required check, and above what a broken walk yields: a walk that saw only src/ finds 16 + // suites and 50 projects, and a non-recursive or mis-patterned enumeration finds none. + private const int DiscoveredSuitesFloor = 20; + private const int ClosureProjectsFloor = 60; + private const int ImplicitInputsFloor = 6; + private const int CrossDirectoryInputsFloor = 3; + + private static readonly string[] ImplicitInputNames = + [ + "Directory.Build.props", "Directory.Build.targets", "Directory.Build.rsp", "Directory.Packages.props", + ".editorconfig", ".globalconfig", "nuget.config", "global.json", + ]; + + // The subset of the implicit inputs that is MSBuild XML and can itself pull files in. + private static readonly string[] ImplicitMsBuildNames = + ["Directory.Build.props", "Directory.Build.targets", "Directory.Packages.props"]; + + // Item types whose Include is an identity and never a file: a project (walked by the closure check + // instead), a package, framework, assembly or namespace name, an assembly attribute, a capability or + // platform, or an analyzer-config name. EVERY other item type is read as naming files. This is a + // denylist on purpose: an allowlist of path types missed a closure project's own Protobuf item, so + // a path-bearing type nobody listed (Protobuf, EditorConfigFiles, a code-generation input) is + // scanned anyway, and a computed path on one fails rather than being skipped. A simple or strong + // assembly name in a Reference holds no directory separator and is skipped as a bare name. + private static readonly HashSet IdentityItemTypes = new(StringComparer.Ordinal) + { + "ProjectReference", "PackageReference", "PackageVersion", "GlobalPackageReference", "PackageDownload", + "FrameworkReference", "InternalsVisibleTo", "AssemblyAttribute", "AssemblyMetadata", "Using", + "ProjectCapability", "SupportedPlatform", "CompilerVisibleProperty", "CompilerVisibleItemMetadata", + }; + + // A path under this property names a file in the NuGet package cache, outside the repository, where + // no glob can reach; the file that names the package and its version is itself covered. The tree's + // one use is the AssembliesToCopy list in src/Ashlar.Tests.Infrastructure/CopyAssemblies.targets. + private const string PackageCacheAnchor = "$(NuGetPackageRoot)"; + + private static readonly Regex ChainImport = new( + @"^\$\(\[MSBuild\]::GetPathOfFileAbove\('(Directory\.Build\.props|Directory\.Build\.targets|Directory\.Packages\.props)'", + RegexOptions.CultureInvariant); + + private readonly ITestOutputHelper _output; + + public ReadinessClosureConventionTests(ITestOutputHelper output) => _output = output; + + [Fact] + public void Every_project_in_a_discovered_suites_closure_is_a_readiness_path() + { + var root = RepoPathResolver.FindRepoRoot(); + var graph = Walk(root); + + graph.Suites.Count.Should().BeGreaterThanOrEqualTo(DiscoveredSuitesFloor, + "validate's sweep discovered {0} test projects; below the stated floor of {1} the discovery " + + "walk is broken and every coverage assertion below would pass on an empty closure", + graph.Suites.Count, DiscoveredSuitesFloor); + graph.ReachedBy.Count.Should().BeGreaterThanOrEqualTo(ClosureProjectsFloor, + "the closure of the discovered suites holds {0} projects; below the stated floor of {1} the " + + "reference walk has stopped following edges", graph.ReachedBy.Count, ClosureProjectsFloor); + graph.Suites.Should().Contain(KernelSuite, "the kernel suite is discovered by validate, not named by any workflow"); + graph.ReachedBy.ContainsKey(KernelDependency).Should().BeTrue( + "the kernel suite tests the SPEC-006 admission code in Ashlar.Manifest (the edge #684 routed)"); + graph.ReachedBy[KernelDependency].Should().Contain(KernelSuite); + graph.AnalyzerOnlyEdges.Should().BeGreaterThan(0, + "Ashlar.Core.Application references Ashlar.Analyzers as an analyzer only; a walk that sees no " + + "such edge is not reading the attribute-order-independent XML"); + graph.SharedFiles.Keys.Should().Contain(new[] { ImplicitSharedFile, ExplicitSharedFile }, + "a ProjectReference declared in a shared MSBuild file is an edge of every project evaluated from it, so the " + + "walk must read the implicit Directory.Build.* files and explicit imports, not only the project files"); + + // Both lists are checked before asserting, so one run names every gap in either list. + var problems = ReadLists(root).SelectMany(x => UncoveredProjects(graph, x.Globs, x.List)).ToList(); + problems.Should().BeEmpty( + "every directory a discovered suite is built from must re-run the lanes that run that suite; " + + "{0} uncovered:\n{1}", problems.Count, string.Join("\n", problems)); + + _output.WriteLine($"suites={graph.Suites.Count} closure={graph.ReachedBy.Count} analyzer-only-edges={graph.AnalyzerOnlyEdges} " + + $"shared-msbuild-files={graph.SharedFiles.Count}"); + foreach (var (project, suites) in graph.ReachedBy) + _output.WriteLine($" {project} <- {suites.Count} suite(s)"); + foreach (var (file, readers) in graph.SharedFiles) + _output.WriteLine($" shared {file} <- {readers.Count} closure project(s)"); + } + + [Fact] + public void Every_implicit_and_cross_directory_build_input_of_that_closure_is_a_readiness_path() + { + var root = RepoPathResolver.FindRepoRoot(); + var graph = Walk(root); + graph.ReachedBy.Count.Should().BeGreaterThanOrEqualTo(ClosureProjectsFloor, + "the inputs below are read from the closure's {0} projects; below the stated floor of {1} the walk " + + "is broken and an empty input set would pass", graph.ReachedBy.Count, ClosureProjectsFloor); + var inputs = BuildInputs(root, graph); + + inputs.Implicit.Count.Should().BeGreaterThanOrEqualTo(ImplicitInputsFloor, + "found {0} implicit build inputs; below the stated floor of {1} the ancestor scan is broken", + inputs.Implicit.Count, ImplicitInputsFloor); + inputs.CrossDirectory.Count.Should().BeGreaterThanOrEqualTo(CrossDirectoryInputsFloor, + "found {0} cross-directory build inputs; below the stated floor of {1} the item scan is broken", + inputs.CrossDirectory.Count, CrossDirectoryInputsFloor); + inputs.Implicit.ContainsKey("src/Directory.Build.props").Should().BeTrue( + "every src/ project in the closure imports src/Directory.Build.props"); + inputs.CrossDirectory.Keys.Should().Contain(key => key.Path.StartsWith("src/Ashlar.Compat/", StringComparison.Ordinal), + "src/Directory.Build.props compiles Ashlar.Compat into most of the closure"); + + var problems = ReadLists(root).SelectMany(x => UncoveredInputs(graph, inputs, x.Globs, x.List)).ToList(); + problems.Should().BeEmpty( + "a file the suites in the closure are built from must re-run the lanes that run them; " + + "{0} uncovered:\n{1}", problems.Count, string.Join("\n", problems)); + + _output.WriteLine($"implicit={inputs.Implicit.Count} cross-directory={inputs.CrossDirectory.Count}"); + foreach (var file in inputs.Implicit.Keys) + _output.WriteLine($" implicit {file}"); + foreach (var (key, declared) in inputs.CrossDirectory) + _output.WriteLine($" cross {key.Path}{(key.IsDirectory ? "/**" : "")} <- {string.Join(", ", declared.Kinds)}"); + } + + [Fact] + public void The_walk_follows_transitive_analyzer_only_and_conditional_references_in_any_attribute_order() + { + using var fixture = new TreeFixture(); + fixture.Write("tests/Suite.Tests/Suite.Tests.csproj", Project("")); + // Include LAST: the shape a '")); + fixture.Write("lib/B/B.csproj", + ""); + fixture.Write("lib/C/C.csproj", ""); + fixture.Write("lib/Unreached/Unreached.csproj", ""); + + var graph = Walk(fixture.Root); + + graph.Suites.Should().Equal("tests/Suite.Tests/Suite.Tests.csproj"); + graph.ReachedBy.Keys.Should().BeEquivalentTo( + new[] { "tests/Suite.Tests/Suite.Tests.csproj", "lib/A/A.csproj", "lib/B/B.csproj", "lib/C/C.csproj" }); + graph.AnalyzerOnlyEdges.Should().Be(1); + } + + [Fact] + public void A_reference_declared_in_a_shared_msbuild_file_is_an_edge_of_every_closure_project_that_reads_it() + { + using var fixture = new TreeFixture(); + // Implicit: every project under lib/ is evaluated from lib/Directory.Build.props. The condition + // names one project; the lens does not evaluate it and gives the edge to every reader. + fixture.Write("lib/Directory.Build.props", + "" + + ""); + // Explicit: the suite imports a targets file that adds an analyzer-only reference, Include last. + fixture.Write("tests/Suite.Tests/Suite.Tests.csproj", + "" + + ""); + fixture.Write("tests/Suite.Tests/Build/Refs.targets", + ""); + fixture.Write("lib/A/A.csproj", ""); + fixture.Write("lib/Shared/Shared.csproj", ""); + fixture.Write("tools/Gen/Gen.csproj", ""); + + var graph = Walk(fixture.Root); + + graph.Suites.Should().Equal("tests/Suite.Tests/Suite.Tests.csproj"); + graph.ReachedBy.Keys.Should().BeEquivalentTo(new[] + { + "tests/Suite.Tests/Suite.Tests.csproj", "lib/A/A.csproj", "lib/Shared/Shared.csproj", "tools/Gen/Gen.csproj", + }, "neither lib/Shared nor tools/Gen is named by any project file; only the shared files reach them"); + graph.ReachedBy["lib/Shared/Shared.csproj"].Should().Equal("tests/Suite.Tests/Suite.Tests.csproj"); + graph.AnalyzerOnlyEdges.Should().Be(1); + graph.SharedFiles.Keys.Should().BeEquivalentTo(new[] { "lib/Directory.Build.props", "tests/Suite.Tests/Build/Refs.targets" }); + graph.SharedFiles["lib/Directory.Build.props"].Should().BeEquivalentTo( + new[] { "lib/A/A.csproj", "lib/Shared/Shared.csproj" }, + "lib/Shared reads the file too; its reference to itself is dropped, not followed or refused"); + + var problems = UncoveredProjects(graph, ["tests/Suite.Tests/**", "lib/A/**", "tools/Gen/**"], "fixture"); + problems.Should().ContainSingle().Which.Should().Contain("'lib/Shared/'").And.Contain("\"lib/Shared/**\""); + } + + [Theory] + [InlineData("Directory.Build.props", // unanchored: resolves against each importing project, which names no one file + "")] + [InlineData("Directory.Build.targets", + "")] + [InlineData("Directory.Build.props", + "")] + [InlineData("tests/Suite.Tests/Refs.targets", // unanchored in an explicit import + "")] + [InlineData("Directory.Build.props", // a wildcard import could pull in a file declaring a reference + "")] + public void A_shared_msbuild_reference_or_import_that_names_no_single_file_is_an_error(string file, string content) + { + using var fixture = new TreeFixture(); + fixture.Write("tests/Suite.Tests/Suite.Tests.csproj", ""); + fixture.Write("lib/Shared/Shared.csproj", ""); + fixture.Write(file, content); + Assert.Throws(() => Walk(fixture.Root)); + } + + [Fact] + public void An_uncovered_closure_directory_is_named_with_the_suite_that_needs_it() + { + using var fixture = new TreeFixture(); + fixture.Write("tests/Suite.Tests/Suite.Tests.csproj", Project("")); + fixture.Write("lib/A/A.csproj", Project("")); + fixture.Write("lib/B/B.csproj", ""); + var graph = Walk(fixture.Root); + + UncoveredProjects(graph, ["tests/Suite.Tests/**", "lib/A/**", "lib/B/**"], "fixture").Should().BeEmpty(); + var problems = UncoveredProjects(graph, ["tests/Suite.Tests/**", "lib/A/**", "lib/B/*"], "fixture"); + problems.Should().ContainSingle("a single * covers only the top level, not a project directory"); + problems[0].Should().Contain("'lib/B/'").And.Contain("tests/Suite.Tests/Suite.Tests.csproj").And.Contain("\"lib/B/**\""); + } + + [Fact] + public void Implicit_imports_and_cross_directory_items_are_found_and_packaging_only_items_are_not() + { + using var fixture = new TreeFixture(); + fixture.Write("Directory.Build.props", + ""); + fixture.Write("lib/Directory.Build.props", + "" + + ""); + fixture.Write("tests/Suite.Tests/Suite.Tests.csproj", Project( + "" + + "" + + "Always" + + "")); + fixture.Write("lib/A/A.csproj", ""); + + var inputs = BuildInputs(fixture.Root, Walk(fixture.Root)); + + inputs.Implicit.Keys.Should().BeEquivalentTo(new[] { "Directory.Build.props", "lib/Directory.Build.props" }); + inputs.CrossDirectory.Keys.Should().BeEquivalentTo( + new[] { new InputKey("shared/Common.cs", false), new InputKey("data", true), new InputKey("notes/Readme.txt", false) }, + "a pack-only None item is not a build input and an item inside the project directory is already covered"); + } + + [Fact] + public void A_cross_directory_path_counts_whatever_its_item_type_and_so_does_a_HintPath() + { + using var fixture = new TreeFixture(); + // A shared file: an anchored path on a type nothing lists counts; a bare name (a file in each + // importer's own directory) and an identity (not a file) neither count nor throw. + fixture.Write("Directory.Build.props", "" + + "" + + "" + + ""); + fixture.Write("tests/Suite.Tests/Suite.Tests.csproj", Project( + "" + + "" + + "" + + "" + + "..\\..\\lib\\Probe.dll" + + "" + // Identities, not files - a computed one included - and a file in the package cache, outside the repository. + + "" + + "" + + "" + + "" + + "")); + + var graph = Walk(fixture.Root); + var inputs = BuildInputs(fixture.Root, graph); + + inputs.CrossDirectory.Keys.Should().BeEquivalentTo(new[] + { + new InputKey("contracts/shared.proto", false), new InputKey("protos/agent.proto", false), + new InputKey("cfg/rules.editorconfig", false), new InputKey("cfg/rules.globalconfig", false), + new InputKey("schemas", true), new InputKey("lib/Probe.dll", false), new InputKey("lib/Direct.dll", false), + }, "a path on an item type nothing lists, or in a HintPath, is a build input exactly as a Compile path is"); + inputs.CrossDirectory[new InputKey("lib/Probe.dll", false)].Kinds.Should().Equal("Reference HintPath"); + var problems = UncoveredInputs(graph, inputs, + ["Directory.Build.props", "tests/Suite.Tests/**", "contracts/**", "cfg/**", "schemas/**", "lib/**"], "fixture"); + problems.Should().ContainSingle().Which.Should().Contain("'protos/agent.proto'").And.Contain("named by Protobuf in") + .And.Contain("tests/Suite.Tests/Suite.Tests.csproj"); + } + + [Theory] + [InlineData("")] + [InlineData("")] + [InlineData("")] + [InlineData("")] + [InlineData("")] + [InlineData("")] + [InlineData("")] // a type nothing lists is still read as a path + [InlineData("$(LibRoot)/Probe.dll")] + public void Unmeasurable_project_shapes_are_errors(string item) + { + using var fixture = new TreeFixture(); + fixture.Write("tests/Suite.Tests/Suite.Tests.csproj", Project(item)); + Assert.Throws(() => BuildInputs(fixture.Root, Walk(fixture.Root))); + } + + [Fact] + public void An_unanchored_item_in_a_shared_import_is_an_error() + { + using var fixture = new TreeFixture(); + fixture.Write("Directory.Build.props", ""); + fixture.Write("tests/Suite.Tests/Suite.Tests.csproj", ""); + Assert.Throws(() => BuildInputs(fixture.Root, Walk(fixture.Root))); + } + + [Theory] + [InlineData("src/AI/**", "src/AI/Nested/Deeper/File.cs", true)] + [InlineData("src/AI/*", "src/AI/Nested/File.cs", false)] + [InlineData("src/AI/**", "src/AIOther/File.cs", false)] + [InlineData("src/Ashlar.Policies/**", "src/Ashlar.Policies.Dev/File.cs", false)] + [InlineData(".editorconfig", ".editorconfig", true)] + public void The_glob_matcher_keeps_GitHub_semantics(string glob, string path, bool expected) + => Matches(glob, path).Should().Be(expected); + + // ---- the walk --------------------------------------------------------------------------------- + + /// The discovered test projects. + /// Each closure project, with the suites whose closure holds it. + /// Distinct analyzer-only (project, target) edges walked. + /// Each shared MSBuild file read for references, with the closure projects evaluated from it. + private sealed record Graph( + IReadOnlyList Suites, + SortedDictionary> ReachedBy, + int AnalyzerOnlyEdges, + SortedDictionary> SharedFiles); + + private readonly record struct InputKey(string Path, bool IsDirectory); + + /// The closure projects and shared MSBuild files that name the path. + /// What names it: an item type, an item type's HintPath, or Import. + private sealed record Declared(SortedSet Files, SortedSet Kinds); + + private sealed record Inputs( + SortedDictionary> Implicit, + SortedDictionary CrossDirectory); + + private static Graph Walk(string root) + { + var projects = EnumerateProjects(root); + var canonical = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var project in projects) + canonical.TryAdd(project, project); + + // The predicate reads the absolute directory; hand it the repo-relative path under a root whose + // own name contains no "test", which is what CI's workspace path is (see the class summary). + var virtualRoot = new DirectoryInfo(Path.Combine(Path.GetPathRoot(Path.GetFullPath(root))!, "readiness-closure-root")); + var suites = projects.Where(project => ValidationServiceAdapter.IsDiscoverableTestProject( + new FileInfo(Path.Combine(virtualRoot.FullName, project)), virtualRoot)).ToArray(); + + var edges = new Dictionary>(StringComparer.Ordinal); + var analyzerOnly = new HashSet<(string, string)>(); + var reachedBy = new SortedDictionary>(StringComparer.Ordinal); + var sharedFiles = new SortedDictionary>(StringComparer.Ordinal); + foreach (var suite in suites) + { + var seen = new HashSet(StringComparer.Ordinal); + var pending = new Stack([suite]); + while (pending.TryPop(out var project)) + { + if (!seen.Add(project)) + continue; + if (!reachedBy.TryGetValue(project, out var by)) + reachedBy[project] = by = new SortedSet(StringComparer.Ordinal); + by.Add(suite); + if (!edges.TryGetValue(project, out var next)) + { + var shared = SharedFilesOf(root, project); + foreach (var file in shared) + { + if (!sharedFiles.TryGetValue(file, out var readers)) + sharedFiles[file] = readers = new SortedSet(StringComparer.Ordinal); + readers.Add(project); + } + edges[project] = next = References(root, project, shared, canonical); + } + foreach (var (target, isAnalyzerOnly) in next) + { + if (isAnalyzerOnly) + analyzerOnly.Add((project, target)); + pending.Push(target); + } + } + } + return new Graph(suites, reachedBy, analyzerOnly.Count, sharedFiles); + } + + /// + /// Every *.csproj below the root, as the sweep's recursive GetFiles would find it. + /// Directories named bin/obj or starting with '.' are pruned: the predicate rejects + /// any project below such a segment, so pruning cannot change the discovered set, and it keeps + /// .git, agent worktrees and build output out of the walk. + /// + private static List EnumerateProjects(string root) + { + var found = new List(); + var pending = new Stack([root]); + while (pending.TryPop(out var directory)) + { + found.AddRange(Directory.EnumerateFiles(directory, "*.csproj") + .Where(file => file.EndsWith(".csproj", StringComparison.Ordinal)) + .Select(file => Relative(root, file))); + foreach (var child in Directory.EnumerateDirectories(directory)) + { + var name = Path.GetFileName(child); + if (name.StartsWith('.') || name.Equals("bin", StringComparison.OrdinalIgnoreCase) + || name.Equals("obj", StringComparison.OrdinalIgnoreCase) + || new DirectoryInfo(child).Attributes.HasFlag(FileAttributes.ReparsePoint)) + continue; + pending.Push(child); + } + } + found.Sort(StringComparer.Ordinal); + return found; + } + + /// + /// The references a project is built with: those in its own file and those in every shared MSBuild + /// file it is evaluated from. A shared file's path must be anchored by + /// $(MSBuildThisFileDirectory); unanchored, MSBuild resolves it against each importing + /// project, so it names a different project per importer and the lens refuses it. + /// + private static IReadOnlyList<(string Target, bool AnalyzerOnly)> References( + string root, string project, IReadOnlyList sharedFiles, IReadOnlyDictionary canonical) + { + var result = new List<(string, bool)>(); + foreach (var file in sharedFiles.Prepend(project)) + { + var isProject = file == project; + foreach (var reference in Load(root, file).Descendants().Where(e => e.Name.LocalName == "ProjectReference")) + { + // Update/Remove forms modify an existing item; only Include adds an edge. + var include = (string?)reference.Attribute("Include"); + if (include is null) + continue; + var analyzerOnly = string.Equals(Metadata(reference, "OutputItemType"), "Analyzer", StringComparison.OrdinalIgnoreCase) + && string.Equals(Metadata(reference, "ReferenceOutputAssembly"), "false", StringComparison.OrdinalIgnoreCase); + foreach (var part in include.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + var path = StripAnchor(part, isProject, out var anchored); + if (!isProject && !anchored) + throw new InvalidDataException($"{file}: ProjectReference '{part}' in a shared MSBuild file resolves against " + + "each importing project; anchor it with $(MSBuildThisFileDirectory) so it names one project"); + if (path.IndexOfAny(['$', '@', '%', '*', '?']) >= 0) + throw new InvalidDataException($"{file}: ProjectReference '{part}' is not a literal path; this lens does not evaluate MSBuild"); + var target = Resolve(DirectoryOf(file), path, file); + if (canonical.TryGetValue(target, out var known)) + target = known; + else if (!File.Exists(Path.Combine(root, target))) + throw new InvalidDataException($"{file}: ProjectReference '{part}' names a project that does not exist ({target})"); + if (target == project) + continue; // a shared file naming one of its own importers: MSBuild refuses the cycle, so a condition excludes it + result.Add((target, analyzerOnly)); + } + } + } + return result; + } + + /// + /// The shared MSBuild files a project is evaluated from besides itself: every + /// Directory.Build.props, Directory.Build.targets and Directory.Packages.props + /// in its directory or an ancestor (an over-approximation - MSBuild imports the nearest of each and + /// reaches farther ones only through a chain import), and every repo file those or the project + /// reach by an explicit <Import>, transitively. + /// + private static List SharedFilesOf(string root, string project) + { + var shared = new List(); + var seen = new HashSet(StringComparer.Ordinal) { project }; + var pending = new Stack(ImplicitFiles(root, project, ImplicitMsBuildNames).Concat(Imports(root, project, isProject: true))); + while (pending.TryPop(out var file)) + { + if (!seen.Add(file)) + continue; + shared.Add(file); + foreach (var import in Imports(root, file, isProject: false)) + pending.Push(import); + } + shared.Sort(StringComparer.Ordinal); + return shared; + } + + /// + /// The repo files an MSBuild file imports explicitly. An Import path resolves against the + /// declaring file. The ancestor chain import and SDK imports are skipped (the first is counted as an + /// implicit file, the second is not in the repo), as is a literal path to a file that does not + /// exist (it contributes nothing to the build); a computed or wildcard path is refused. + /// + private static IEnumerable Imports(string root, string file, bool isProject) + { + foreach (var element in Load(root, file).Descendants().Where(e => e.Name.LocalName == "Import")) + { + var value = (string?)element.Attribute("Project"); + if (value is null || element.Attribute("Sdk") is not null) + continue; + foreach (var part in value.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + if (ChainImport.IsMatch(part)) + continue; + var path = StripAnchor(part, isProject, out _); + if (path.IndexOfAny(['$', '@', '%', '*', '?']) >= 0) + throw new InvalidDataException($"{file}: Import '{part}' is not a literal path; this lens does not evaluate " + + "MSBuild or expand wildcards, and an imported file can declare references"); + var target = Resolve(DirectoryOf(file), path, file); + if (File.Exists(Path.Combine(root, target))) + yield return target; + } + } + } + + /// Files with one of in the project's directory or any ancestor up to the root. + private static IEnumerable ImplicitFiles(string root, string project, string[] names) + { + var directory = DirectoryOf(project); + while (true) + { + foreach (var file in Directory.EnumerateFiles(Path.Combine(root, directory)).Order(StringComparer.Ordinal)) + { + var name = Path.GetFileName(file); + if (names.Contains(name, StringComparer.OrdinalIgnoreCase)) + yield return Join(directory, name); + } + if (directory.Length == 0) + yield break; + directory = DirectoryOf(directory); + } + } + + /// + /// Strips a leading $(MSBuildThisFileDirectory) (the declaring file's directory) or, in a + /// project file, $(MSBuildProjectDirectory); says whether one was. + /// + private static string StripAnchor(string part, bool isProject, out bool anchored) + { + const string ThisFile = "$(MSBuildThisFileDirectory)"; + const string ProjectDirectory = "$(MSBuildProjectDirectory)"; + anchored = true; + if (part.StartsWith(ThisFile, StringComparison.Ordinal)) + return part[ThisFile.Length..].TrimStart('/', '\\'); + if (isProject && part.StartsWith(ProjectDirectory, StringComparison.Ordinal)) + return part[ProjectDirectory.Length..].TrimStart('/', '\\'); + anchored = false; + return part; + } + + // ---- implicit and cross-directory inputs ------------------------------------------------------ + + private static Inputs BuildInputs(string root, Graph graph) + { + var implicitInputs = new SortedDictionary>(StringComparer.Ordinal); + foreach (var project in graph.ReachedBy.Keys) + foreach (var relative in ImplicitFiles(root, project, ImplicitInputNames)) + { + if (!implicitInputs.TryGetValue(relative, out var importers)) + implicitInputs[relative] = importers = new SortedSet(StringComparer.Ordinal); + importers.Add(project); + } + + var cross = new SortedDictionary(Comparer.Create((a, b) => + { + var byPath = string.CompareOrdinal(a.Path, b.Path); + return byPath != 0 ? byPath : a.IsDirectory.CompareTo(b.IsDirectory); + })); + void Add(InputKey key, string declaredBy, string kind) + { + if (!cross.TryGetValue(key, out var by)) + cross[key] = by = new Declared(new SortedSet(StringComparer.Ordinal), new SortedSet(StringComparer.Ordinal)); + by.Files.Add(declaredBy); + by.Kinds.Add(kind); + } + foreach (var project in graph.ReachedBy.Keys) + foreach (var (key, kind) in ItemInputs(root, project, isProject: true)) + Add(key, project, kind); + // The implicit MSBuild files above closure projects and every file they or a project import. + foreach (var file in graph.SharedFiles.Keys) + foreach (var (key, kind) in ItemInputs(root, file, isProject: false)) + Add(key, file, kind); + return new Inputs(implicitInputs, cross); + } + + /// + /// The paths outside the declaring project that an MSBuild file names: the Include of every item + /// whose type names files (), the HintPath of any item, and every Import. + /// MSBuild resolves an ITEM path against the project being built and an Import against the + /// declaring file; for a project file both are its own directory, and in a shared import only a + /// path anchored by $(MSBuildThisFileDirectory) names one file for every importer - except a + /// bare file name, which names a file in each importer's own directory. + /// + private static IEnumerable<(InputKey Key, string Kind)> ItemInputs(string root, string file, bool isProject) + { + var fileDirectory = DirectoryOf(file); + foreach (var element in Load(root, file).Descendants()) + { + var kind = element.Name.LocalName; + var values = new List<(string Label, string Value)>(); + if (kind == "Import") + { + if (element.Attribute("Sdk") is null && (string?)element.Attribute("Project") is { } project) + values.Add((kind, project)); + } + else if (element.Parent?.Name.LocalName == "ItemGroup") + { + if ((string?)element.Attribute("Include") is { } include && NamesFiles(element)) + values.Add((kind, include)); + if (Metadata(element, "HintPath") is { } hintPath) + values.Add((kind + " HintPath", hintPath)); + } + foreach (var (label, value) in values) + foreach (var part in value.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + if (kind == "Import" && ChainImport.IsMatch(part)) + continue; // the ancestor chain, already counted as an implicit input + if (part.StartsWith(PackageCacheAnchor, StringComparison.Ordinal)) + continue; // the NuGet package cache, outside the repository + if (kind != "Import" && part.IndexOfAny(['/', '\\', '$', '@', '%']) < 0 && part is not ("." or "..")) + continue; // a bare name: a file in the project's own directory, or in each importer's, which its glob covers + var path = StripAnchor(part, isProject, out var anchored); + if (path.Contains("$(", StringComparison.Ordinal) || path.Contains("@(", StringComparison.Ordinal) + || path.Contains("%(", StringComparison.Ordinal)) + throw new InvalidDataException($"{file}: {label} '{part}' is not a literal path; this lens does not " + + "evaluate MSBuild (an item type whose Include names no file belongs in IdentityItemTypes)"); + if (!anchored && kind != "Import" && !isProject) + throw new InvalidDataException($"{file}: {label} '{part}' resolves against each importing project; " + + "anchor it with $(MSBuildThisFileDirectory) so it names one file"); + var segments = path.Replace('\\', '/').Split('/'); + var wildcard = Array.FindIndex(segments, segment => segment.IndexOfAny(['*', '?']) >= 0); + var literal = string.Join('/', wildcard < 0 ? segments : segments.Take(wildcard)); + var target = Resolve(fileDirectory, literal, file); + if (isProject && IsWithin(target, fileDirectory)) + continue; // covered by the project's own directory glob + yield return (new InputKey(target, wildcard >= 0), label); + } + } + } + + /// + /// Whether an item's Include names files: every type but an identity type, and a None item only + /// when it is copied to the output (a pack-only README is a packaging input, not a build or test input). + /// + private static bool NamesFiles(XElement item) + { + var kind = item.Name.LocalName; + return !IdentityItemTypes.Contains(kind) && (kind != "None" || CopiesToOutput(item)); + } + + private static bool CopiesToOutput(XElement item) + { + var copy = Metadata(item, "CopyToOutputDirectory"); + return copy is not null && !copy.Trim().Equals("Never", StringComparison.OrdinalIgnoreCase); + } + + // ---- coverage --------------------------------------------------------------------------------- + + private static List UncoveredProjects(Graph graph, IReadOnlyList globs, string list) + { + var problems = new List(); + foreach (var (project, suites) in graph.ReachedBy) + { + var directory = DirectoryOf(project); + if (!CoversDirectory(globs, directory, Path.GetFileName(project))) + problems.Add($"{list}: project directory '{directory}/' ({Path.GetFileName(project)}) is built into " + + $"{suites.Count} discovered suite(s), e.g. {suites.Min}, but no glob covers it; add \"{directory}/**\" " + + "to on.push.paths and READINESS_PATHS at the same position"); + } + return problems; + } + + private static List UncoveredInputs(Graph graph, Inputs inputs, IReadOnlyList globs, string list) + { + // A closure project's first suite, or, for a shared MSBuild file, the first suite of its first reader. + string SuiteOf(string declaredBy) => graph.ReachedBy.TryGetValue(declaredBy, out var suites) + ? suites.Min! : SuiteOf(graph.SharedFiles[declaredBy].Min!); + var problems = new List(); + foreach (var (file, importers) in inputs.Implicit) + if (!globs.Any(glob => Matches(glob, file))) + problems.Add($"{list}: '{file}' is an implicit build input of {importers.Min} (in the closure of " + + $"{SuiteOf(importers.Min!)}) but no glob covers it; add \"{file}\" to both lists"); + foreach (var (key, declared) in inputs.CrossDirectory) + { + var covered = key.IsDirectory + ? CoversDirectory(globs, key.Path, "Probe.cs") + : globs.Any(glob => Matches(glob, key.Path)); + if (!covered) + problems.Add($"{list}: '{key.Path}{(key.IsDirectory ? "/**" : "")}' is named by {string.Join(", ", declared.Kinds)} in " + + $"{declared.Files.Min} (in the closure of {SuiteOf(declared.Files.Min!)}) but no glob covers it"); + } + return problems; + } + + /// A directory is covered when one glob matches both a top-level and a deeply nested file in it. + private static bool CoversDirectory(IReadOnlyList globs, string directory, string leaf) + => globs.Any(glob => Matches(glob, Join(directory, leaf)) && Matches(glob, Join(directory, "Nested/Deeper/Probe.cs"))); + + /// + /// Mirror of AiPipelineCiRoutingConventionTests.Matches: GitHub glob semantics for the + /// shapes the readiness lists use (literals, basename *, a single trailing /**). + /// GitHub's single * does not cross '/', unlike Bash's in the in-job matcher. + /// + private static bool Matches(string glob, string path) + { + if (!Regex.IsMatch(glob, @"^[A-Za-z0-9_./*\-]+$") || glob.Contains("***", StringComparison.Ordinal) + || (glob.Contains("**", StringComparison.Ordinal) && glob != "**" + && (!glob.EndsWith("/**", StringComparison.Ordinal) + || glob.IndexOf("**", StringComparison.Ordinal) != glob.Length - 2))) + throw new InvalidDataException($"Unsupported path glob: {glob}"); + var pattern = Regex.Escape(glob).Replace(@"\*\*", ".*").Replace(@"\*", "[^/]*"); + return Regex.IsMatch(path, "\\A" + pattern + "\\z", RegexOptions.CultureInvariant); + } + + // ---- the two readiness lists ------------------------------------------------------------------ + + private static IEnumerable<(string List, IReadOnlyList Globs)> ReadLists(string root) + { + var text = File.ReadAllText(Path.Combine(root, Readiness)).Replace("\r\n", "\n"); + YamlMappingNode workflow; + try + { + var yaml = new YamlStream(); + yaml.Load(new StringReader(text)); + workflow = (YamlMappingNode)yaml.Documents.Single().RootNode; + } + catch (YamlException ex) { throw new InvalidDataException("Unreadable readiness workflow YAML", ex); } + + var push = ((YamlSequenceNode)At(workflow, "on", "push", "paths")).Children + .Select(node => ((YamlScalarNode)node).Value!).ToArray(); + var filter = ((YamlSequenceNode)At(workflow, "jobs", "changes", "steps")).Children.Cast() + .Single(step => step.Children.TryGetValue(new YamlScalarNode("id"), out var id) && ((YamlScalarNode)id).Value == "filter"); + var internalPaths = ReadShellArray(((YamlScalarNode)At(filter, "run")).Value!); + + push.Should().NotBeEmpty(); + internalPaths.Should().NotBeEmpty(); + yield return ("on.push.paths", push); + yield return ("READINESS_PATHS", internalPaths); + } + + private static YamlNode At(YamlNode node, params string[] keys) + { + foreach (var key in keys) + node = ((YamlMappingNode)node).Children.TryGetValue(new YamlScalarNode(key), out var next) + ? next : throw new InvalidDataException($"Missing YAML key: {key}"); + return node; + } + + /// + /// The literal array's quoted entries. AiPipelineCiRoutingConventionTests owns the strict + /// shape checks (one literal declaration, no other writes); this reader refuses what it cannot read. + /// + private static string[] ReadShellArray(string script) + { + var lines = script.Replace("\r\n", "\n").Split('\n'); + var start = Array.FindIndex(lines, line => line.Trim() == "READINESS_PATHS=("); + if (start < 0) + throw new InvalidDataException("No literal READINESS_PATHS array"); + var result = new List(); + foreach (var raw in lines.Skip(start + 1)) + { + var line = raw.Trim(); + if (line == ")") + return result.ToArray(); + if (line.Length == 0 || line.StartsWith('#')) + continue; + var match = Regex.Match(line, "^\"([^\"]+)\"$"); + if (!match.Success) + throw new InvalidDataException($"Unsupported READINESS_PATHS entry: {line}"); + result.Add(match.Groups[1].Value); + } + throw new InvalidDataException("Unterminated READINESS_PATHS"); + } + + // ---- paths ------------------------------------------------------------------------------------ + + private static XDocument Load(string root, string relative) + { + try { return XDocument.Load(Path.Combine(root, relative)); } + catch (System.Xml.XmlException ex) { throw new InvalidDataException($"{relative}: unreadable MSBuild XML", ex); } + } + + private static string? Metadata(XElement item, string name) + => (string?)item.Attribute(name) ?? item.Elements().FirstOrDefault(e => e.Name.LocalName == name)?.Value; + + /// Resolves a relative path against a repo-relative directory, refusing absolute or escaping paths. + private static string Resolve(string baseDirectory, string relative, string declaredIn) + { + var normalized = relative.Replace('\\', '/'); + if (normalized.StartsWith('/') || (normalized.Length > 1 && normalized[1] == ':')) + throw new InvalidDataException($"{declaredIn}: '{relative}' is an absolute path"); + var segments = baseDirectory.Length == 0 ? new List() : baseDirectory.Split('/').ToList(); + foreach (var segment in normalized.Split('/')) + { + if (segment is "" or ".") + continue; + if (segment == "..") + { + if (segments.Count == 0) + throw new InvalidDataException($"{declaredIn}: '{relative}' leaves the repository"); + segments.RemoveAt(segments.Count - 1); + } + else + segments.Add(segment); + } + return string.Join('/', segments); + } + + private static bool IsWithin(string path, string directory) + => directory.Length == 0 || path == directory || path.StartsWith(directory + "/", StringComparison.Ordinal); + + private static string DirectoryOf(string relative) + { + var slash = relative.LastIndexOf('/'); + return slash < 0 ? "" : relative[..slash]; + } + + private static string Join(string directory, string leaf) => directory.Length == 0 ? leaf : directory + "/" + leaf; + + private static string Relative(string root, string path) => Path.GetRelativePath(root, path).Replace('\\', '/'); + + private static string Project(string items) => $"{items}"; + + private sealed class TreeFixture : IDisposable + { + public string Root { get; } = Path.Combine(Path.GetTempPath(), "readiness-closure-" + Guid.NewGuid().ToString("N")); + + public void Write(string relative, string content) + { + var path = Path.Combine(Root, relative); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, content); + } + + public void Dispose() + { + if (Directory.Exists(Root)) + Directory.Delete(Root, recursive: true); + } + } +} diff --git a/src/Ashlar.Tests.Infrastructure/Tests/Certification/TestOwnershipRunnerConventionTests.cs b/src/Ashlar.Tests.Infrastructure/Tests/Certification/TestOwnershipRunnerConventionTests.cs index 9e3e18604..fc0bba989 100644 --- a/src/Ashlar.Tests.Infrastructure/Tests/Certification/TestOwnershipRunnerConventionTests.cs +++ b/src/Ashlar.Tests.Infrastructure/Tests/Certification/TestOwnershipRunnerConventionTests.cs @@ -219,9 +219,9 @@ public void Every_named_runner_resolves_to_a_gate_definition_on_disk() + "the walk is recursive for exactly this reason)\n" + " - a Makefile target header, or a name on the .PHONY line at Makefile:1 (the " + "UNION of the two: 110 .PHONY tokens, 140 target headers)\n" - + " kernel-gate -> Makefile:194\n" + + " kernel-gate -> the kernel-gate: header in Makefile\n" + " - a job id in any .github/workflows/*.yml\n" - + " readiness-summary -> full-platform-readiness-gate.yml:586, which carries the " + + " readiness-summary -> the job id in full-platform-readiness-gate.yml, which carries the " + "required 'Readiness summary' check and has no file of its own\n" + "Matching is on the BASENAME, never a workflow's display name: cert-gate.yml " + "declares 'name: Cert gate' and mcp-a2a-gate.yml declares 'name: MCP + A2A protocol " diff --git a/src/Ashlar.Tests.Infrastructure/Tests/Validation/ValidationServiceAdapterGapCoverageTests.cs b/src/Ashlar.Tests.Infrastructure/Tests/Validation/ValidationServiceAdapterGapCoverageTests.cs index 7a6994e0e..9e8f5a2ca 100644 --- a/src/Ashlar.Tests.Infrastructure/Tests/Validation/ValidationServiceAdapterGapCoverageTests.cs +++ b/src/Ashlar.Tests.Infrastructure/Tests/Validation/ValidationServiceAdapterGapCoverageTests.cs @@ -283,8 +283,16 @@ public async Task ValidateAsync_rejects_a_parser_that_drops_recorded_rows() result.TestsRun.Should().Be(0); result.TestsPassed.Should().Be(0); result.TestsFailed.Should().Be(0); - result.EvidenceErrors.Should().ContainSingle().Which.Should().Contain("PassTests.csproj"); + result.EvidenceErrors.Should().ContainSingle().Which.Should().ContainAll( + "PassTests.csproj", "test results are missing, unreadable or inconsistent"); parser.Verify(p => p.ParseAsync(It.IsAny(), It.IsAny()), Times.Once); + + // Invalid evidence is judged before, and apart from, the zero-execution rule, so a + // caller's filter - which switches that rule off - cannot turn it into a pass. + var filtered = await adapter.ValidateAsync("FullyQualifiedName~PassTests", progress: null, CancellationToken.None); + filtered.Passed.Should().BeFalse("a filter does not excuse unreadable evidence: " + filtered.Message); + filtered.EvidenceErrors.Should().ContainSingle().Which.Should().ContainAll( + "PassTests.csproj", "test results are missing, unreadable or inconsistent"); } finally { @@ -456,26 +464,176 @@ public async Task Missing_or_corrupt_results_fail_without_inventing_a_test(bool } } + /// + /// A discovered project whose every test is Category=Stress selects nothing under + /// validate's default exclusions. Until 2026-09-30 this fact was + /// An_empty_selection_passes_with_zero_observed_tests and pinned that as a PASS, so an + /// unfiltered sweep could report green for a project it ran nothing in. It was flipped + /// deliberately, and the policy it now pins is the conservative one: the project FAILS even + /// though its empty selection is legitimate, because a sweep that reports "passed" for a + /// project it executed nothing in is a lane, not a receipt. A future all-Stress or + /// all-DockerOptional test project must carry at least one test outside those categories (a + /// fast structural or smoke fact); there is no opt-out marker. A blank filter is no filter, + /// exactly as it is for the argv validate builds. Only a caller's real filter chooses the + /// selection, and under one the same empty project still passes. + /// [Fact(Timeout = TestTimeouts.E2E)] - public async Task An_empty_selection_passes_with_zero_observed_tests() + public async Task An_unfiltered_sweep_fails_a_project_whose_every_test_is_excluded_by_default() { var original = Directory.GetCurrentDirectory(); var temp = CreatePassingTestProjectDir(); var source = Path.Combine(temp, "tests", "PassTests.cs"); File.WriteAllText(source, File.ReadAllText(source).Replace("[Fact]", "[Fact, Trait(\"Category\", \"Stress\")]", StringComparison.Ordinal)); + var arranged = File.ReadAllText(source); + arranged.Should().NotContain("[Fact]", "the arrange step must have tagged every fact"); + Occurrences(arranged, "Trait(\"Category\", \"Stress\")").Should().Be(2, "both fixture facts must be Stress-only"); + try + { + Directory.SetCurrentDirectory(temp); + var adapter = new ValidationServiceAdapter(NullLogger.Instance, + new TrxTestResultParser(NullLogger.Instance)); + + foreach (var unfiltered in new string?[] { null, " " }) + { + var because = $"filter {(unfiltered is null ? "null" : "'" + unfiltered + "'")} is no filter"; + var result = await adapter.ValidateAsync(unfiltered); + result.Passed.Should().BeFalse($"{because}, and the only discovered project executed nothing"); + result.EvidenceErrors.Should().ContainSingle(because).Which.Should().ContainAll( + "PassTests.csproj", "executed no tests in an unfiltered sweep", "it selected none"); + result.TestsRun.Should().Be(0, because); + result.TestsPassed.Should().Be(0, because); + result.TestsFailed.Should().Be(0, "an empty project is not a failed test"); + result.TestsSkipped.Should().Be(0, because); + result.TestResults.Should().BeEmpty("an evidence failure is not an executed test"); + result.Message.Should().StartWith( + "Validation failed (0/0 tests passed, but 1 project produced no passing evidence); " + + "PassTests.csproj: executed no tests in an unfiltered sweep", because); + result.Message.Should().NotContain("no tests selected", "the project is reported once, as a failure"); + } + + var narrowed = await adapter.ValidateAsync("Category=Stress"); + narrowed.Passed.Should().BeTrue("a caller's filter chose this selection: " + narrowed.Message); + narrowed.EvidenceErrors.Should().BeEmpty(); + narrowed.TestsRun.Should().Be(0); + narrowed.Message.Should().Be("Validation passed (0/0 tests); no tests selected: PassTests.csproj"); + } + finally + { + Directory.SetCurrentDirectory(original); + Directory.Delete(temp, recursive: true); + } + } + + /// + /// The arm a check keyed on NoTestsSelected is blind to: every selected test is skipped, + /// so the run records rows, classifies as ResultsRecorded, and executed nothing. The + /// rule is "executed == 0", not "selected == 0". A project with SOME executed tests beside its + /// skips still passes (ValidateAsync_does_not_count_skipped_tests_as_failures). This is + /// also the shape of a suite made only of [OptInFact] tests without their switch, or + /// only of [NotOnCiFact] tests under CI=true: it fails on the lane where it + /// executes nothing, and must carry one test that runs there without the dependency. + /// + [Fact(Timeout = TestTimeouts.E2E)] + public async Task An_unfiltered_sweep_fails_a_project_whose_every_selected_test_is_skipped() + { + var original = Directory.GetCurrentDirectory(); + var temp = CreatePassingTestProjectDir(); + var source = Path.Combine(temp, "tests", "PassTests.cs"); + File.WriteAllText(source, File.ReadAllText(source).Replace("[Fact]", "[Fact(Skip = \"documented gap\")]", StringComparison.Ordinal)); + var arranged = File.ReadAllText(source); + arranged.Should().NotContain("[Fact]", "the arrange step must have skipped every fact"); + Occurrences(arranged, "Skip = \"documented gap\"").Should().Be(2, "both fixture facts must be skipped"); try { Directory.SetCurrentDirectory(temp); var adapter = new ValidationServiceAdapter(NullLogger.Instance, new TrxTestResultParser(NullLogger.Instance)); + var result = await adapter.ValidateAsync(null); - result.Passed.Should().BeTrue(); - result.EvidenceErrors.Should().BeEmpty(); + result.TestsSkipped.Should().Be(2, + "both rows were recorded as skipped, so this run took the ResultsRecorded arm, not NoTestsSelected: " + + result.Message); + result.Passed.Should().BeFalse("the only discovered project executed nothing"); + result.EvidenceErrors.Should().ContainSingle().Which.Should().ContainAll( + "PassTests.csproj", "executed no tests in an unfiltered sweep", "all 2 of its selected tests were skipped"); result.TestsRun.Should().Be(0); - result.TestsPassed.Should().Be(0); - result.TestsFailed.Should().Be(0); + result.TestsFailed.Should().Be(0, "a skipped project is not a failed test"); + result.TestResults.Should().BeEmpty("an evidence failure is not an executed test"); + result.Message.Should().NotContain("no tests selected"); + + var narrowed = await adapter.ValidateAsync("FullyQualifiedName~PassTests"); + narrowed.Passed.Should().BeTrue("a caller's filter chose this selection: " + narrowed.Message); + narrowed.EvidenceErrors.Should().BeEmpty(); + narrowed.TestsSkipped.Should().Be(2); + narrowed.Message.Should().Be("Validation passed (0/0 tests, 2 skipped)"); + } + finally + { + Directory.SetCurrentDirectory(original); + Directory.Delete(temp, recursive: true); + } + } + + /// + /// The other empty-selection arm. ClassifyCompletedRun reaches NoTestsSelected + /// two ways: a TRX with zero rows (the Stress-only fact above), or NO TRX under the project + /// directory beside a console line saying the filter matched nothing. VSTest 17.12 writes a + /// zero-row TRX even for an empty selection, so a fixture that leaves the TRX in place never + /// reaches the second arm, and a rule tied to TRX presence passed every other fact. Here the + /// project sends its TRX outside its own directory through VSTestResultsDirectory (the + /// shape a repository-level Directory.Build.props commonly has), so validate finds none and + /// must classify the run from the console alone. The zero-execution rule may not depend on + /// which arm classified the run: unfiltered, the project fails; under a caller's filter the + /// same empty selection passes and is reported as one. + /// + [Fact(Timeout = TestTimeouts.E2E)] + public async Task An_unfiltered_sweep_fails_an_empty_selection_that_left_no_trx_in_the_project() + { + var original = Directory.GetCurrentDirectory(); + var temp = CreatePassingTestProjectDir(); + var projectDir = Path.Combine(temp, "tests"); + var redirected = Path.Combine(temp, "trx-out"); + var source = Path.Combine(projectDir, "PassTests.cs"); + File.WriteAllText(source, File.ReadAllText(source).Replace("[Fact]", "[Fact, Trait(\"Category\", \"Stress\")]", StringComparison.Ordinal)); + var project = Path.Combine(projectDir, "PassTests.csproj"); + File.WriteAllText(project, File.ReadAllText(project).Replace( + "false", + "false$(MSBuildProjectDirectory)/../trx-out", + StringComparison.Ordinal)); + var arrangedSource = File.ReadAllText(source); + arrangedSource.Should().NotContain("[Fact]", "the arrange step must have tagged every fact"); + Occurrences(arrangedSource, "Trait(\"Category\", \"Stress\")").Should().Be(2, "both fixture facts must be Stress-only"); + Occurrences(File.ReadAllText(project), "").Should().Be(1, + "the arrange step must have redirected the project's results directory"); + try + { + Directory.SetCurrentDirectory(temp); + var adapter = new ValidationServiceAdapter(NullLogger.Instance, + new TrxTestResultParser(NullLogger.Instance)); + + var result = await adapter.ValidateAsync(null); + var redirectedTrx = Directory.Exists(redirected) + ? Directory.GetFiles(redirected, "*.trx", SearchOption.AllDirectories).Length + : 0; + Directory.GetFiles(projectDir, "*.trx", SearchOption.AllDirectories).Should().BeEmpty( + "the run must leave no TRX where validate looks, or it took the zero-row TRX arm instead " + + $"({redirectedTrx} TRX file(s) were redirected): {result.Message}"); + result.Passed.Should().BeFalse("filter null is no filter, and the only discovered project executed nothing"); + result.EvidenceErrors.Should().ContainSingle(result.Message).Which.Should().ContainAll( + "PassTests.csproj", "executed no tests in an unfiltered sweep", "it selected none"); + result.TestsRun.Should().Be(0); + result.TestsFailed.Should().Be(0, "an empty project is not a failed test"); result.TestsSkipped.Should().Be(0); - result.Message.Should().Contain("no tests selected"); + result.TestResults.Should().BeEmpty("an evidence failure is not an executed test"); + result.Message.Should().NotContain("no tests selected", "the project is reported once, as a failure"); + + var narrowed = await adapter.ValidateAsync("Category=Stress"); + Directory.GetFiles(projectDir, "*.trx", SearchOption.AllDirectories).Should().BeEmpty( + "the filtered run must take the same no-TRX arm: " + narrowed.Message); + narrowed.Passed.Should().BeTrue("a caller's filter chose this selection: " + narrowed.Message); + narrowed.EvidenceErrors.Should().BeEmpty(); + narrowed.TestsRun.Should().Be(0); + narrowed.Message.Should().Be("Validation passed (0/0 tests); no tests selected: PassTests.csproj"); } finally { @@ -484,6 +642,140 @@ public async Task An_empty_selection_passes_with_zero_observed_tests() } } + /// + /// The zero-execution rule is judged per project, on that project's OWN executed count. Every + /// other E2E fixture here discovers exactly one project, where the sweep's running total always + /// equals that project's count, so a rule judged on the running total (or on the result list, + /// or only for the first project) passed all of them - and would reopen the gap in a real + /// many-project sweep for any empty project discovered after one that executed something. Here + /// an executing project is discovered FIRST and an all-Stress project SECOND, so the total is + /// already 2 when the empty one is judged, and the sweep must still fail on it. The order is + /// deterministic: file enumeration yields a directory's own files before it descends into a + /// subdirectory, and the arrange step asserts that order through the adapter's own discovery. + /// + [Fact(Timeout = TestTimeouts.E2E)] + public async Task An_unfiltered_sweep_fails_an_empty_project_discovered_after_one_that_executed_tests() + { + var original = Directory.GetCurrentDirectory(); + var temp = CreatePassingTestProjectDir(); + var outer = Path.Combine(temp, "tests"); + var inner = Path.Combine(outer, "zz"); + Directory.CreateDirectory(inner); + var passProject = Path.Combine(outer, "PassTests.csproj"); + var projectXml = File.ReadAllText(passProject); + File.WriteAllText(Path.Combine(inner, "EmptyTests.csproj"), projectXml); + File.WriteAllText(passProject, projectXml.Replace( + "", "", StringComparison.Ordinal)); + var emptySource = Path.Combine(inner, "EmptyTests.cs"); + File.WriteAllText(emptySource, """ + using Xunit; + public class EmptyTests + { + [Fact, Trait("Category", "Stress")] public void OnlyStress() { } + } + """); + + var root = new DirectoryInfo(temp); + root.GetFiles("*.csproj", SearchOption.AllDirectories) + .Where(f => ValidationServiceAdapter.IsDiscoverableTestProject(f, root)) + .Select(f => f.Name) + .Should().Equal(new[] { "PassTests.csproj", "EmptyTests.csproj" }, + "the executing project must be discovered first, so a running total is already > 0 when the empty one is judged"); + Occurrences(File.ReadAllText(passProject), "").Should().Be(1, + "the outer project must not compile the inner project's sources"); + var arrangedEmpty = File.ReadAllText(emptySource); + arrangedEmpty.Should().NotContain("[Fact]", "the inner project's only fact must be tagged"); + Occurrences(arrangedEmpty, "Trait(\"Category\", \"Stress\")").Should().Be(1, "the inner project must be Stress-only"); + try + { + Directory.SetCurrentDirectory(temp); + var adapter = new ValidationServiceAdapter(NullLogger.Instance, + new TrxTestResultParser(NullLogger.Instance)); + + var result = await adapter.ValidateAsync(null); + result.TestsRun.Should().Be(2, "the first project executed both of its facts: " + result.Message); + result.TestsPassed.Should().Be(2, result.Message); + result.TestsFailed.Should().Be(0, "an empty project is not a failed test: " + result.Message); + result.TestsSkipped.Should().Be(0, result.Message); + result.TestResults.Should().NotBeNull(); + result.TestResults!.Select(test => test.Name).Should().BeEquivalentTo( + new[] { "PassTests.Ok", "PassTests.Other" }, "only the outer project executed anything"); + result.Passed.Should().BeFalse( + "EmptyTests.csproj executed nothing in an unfiltered sweep, whatever ran before it: " + result.Message); + result.EvidenceErrors.Should().ContainSingle(result.Message).Which.Should().StartWith( + "EmptyTests.csproj: executed no tests in an unfiltered sweep (it selected none"); + result.Message.Should().StartWith( + "Validation failed (2/2 tests passed, but 1 project produced no passing evidence); " + + "EmptyTests.csproj: executed no tests in an unfiltered sweep"); + result.Message.Should().NotContain("no tests selected", "the empty project is reported once, as a failure"); + } + finally + { + Directory.SetCurrentDirectory(original); + Directory.Delete(temp, recursive: true); + } + } + + [Theory] + [InlineData(null, 0, 0, true)] + [InlineData("", 0, 0, true)] + [InlineData(" ", 0, 0, true)] + [InlineData(" \t\r\n", 0, 0, true)] + [InlineData(null, 0, 1, true)] + [InlineData(null, 0, 7, true)] + [InlineData("", 0, 3, true)] + [InlineData(null, 1, 0, false)] + [InlineData(null, 1, 5, false)] + [InlineData("Category=First", 0, 0, false)] + [InlineData("Category=First", 0, 3, false)] + public void Only_an_unfiltered_project_that_executed_nothing_is_an_evidence_error( + string? filter, int executed, int skipped, bool fails) + { + var error = ValidationServiceAdapter.ZeroExecutionError("Some.Tests.csproj", filter, executed, skipped); + if (fails) + error.Should().NotBeNull().And.StartWith("Some.Tests.csproj: executed no tests in an unfiltered sweep ("); + else + error.Should().BeNull(); + } + + [Theory] + [InlineData(0, "it selected none, and validate always excludes Category=Stress and Category=DockerOptional")] + [InlineData(1, "its only selected test was skipped")] + [InlineData(4, "all 4 of its selected tests were skipped")] + public void The_zero_execution_error_says_whether_nothing_was_selected_or_everything_was_skipped( + int skipped, string reason) + => ValidationServiceAdapter.ZeroExecutionError("Some.Tests.csproj", null, 0, skipped).Should().Be( + $"Some.Tests.csproj: executed no tests in an unfiltered sweep ({reason}); " + + "a discovered test project that runs nothing is not evidence of passing."); + + [Fact] + public void An_evidence_only_failure_does_not_headline_as_zero_tests_failed() + { + var zero = ValidationServiceAdapter.ZeroExecutionError("HelloBrick.Tests.csproj", null, 0, 0)!; + var message = ValidationServiceAdapter.DescribeOutcome( + false, 5703, 5703, 0, 13, new[] { zero }, Array.Empty()); + message.Should().Be( + "Validation failed (5703/5703 tests passed, 13 skipped, but 1 project produced no passing evidence); " + zero); + message.Should().NotContain("tests failed"); + + ValidationServiceAdapter.DescribeOutcome(false, 2, 2, 0, 0, new[] { "A: x", "B: y" }, Array.Empty()) + .Should().Be("Validation failed (2/2 tests passed, but 2 projects produced no passing evidence); A: x; B: y"); + } + + [Fact] + public void A_failed_test_still_headlines_the_failed_count_and_a_pass_lists_empty_selections() + { + ValidationServiceAdapter.DescribeOutcome(false, 10, 9, 1, 2, Array.Empty(), Array.Empty()) + .Should().Be("Validation failed (1/10 tests failed, 2 skipped)"); + ValidationServiceAdapter.DescribeOutcome(false, 10, 9, 1, 0, new[] { "A: x" }, Array.Empty()) + .Should().Be("Validation failed (1/10 tests failed); A: x"); + ValidationServiceAdapter.DescribeOutcome(true, 3, 3, 0, 0, Array.Empty(), new[] { "A.csproj", "B.csproj" }) + .Should().Be("Validation passed (3/3 tests); no tests selected: A.csproj, B.csproj"); + } + + private static int Occurrences(string text, string token) + => (text.Length - text.Replace(token, string.Empty, StringComparison.Ordinal).Length) / token.Length; + [Fact(Timeout = TestTimeouts.E2E)] public async Task Caller_filter_selects_observed_results_and_preserves_default_exclusions() { diff --git a/tests/scripts/cert-gate-guards.test.sh b/tests/scripts/cert-gate-guards.test.sh new file mode 100644 index 000000000..0d33eb086 --- /dev/null +++ b/tests/scripts/cert-gate-guards.test.sh @@ -0,0 +1,334 @@ +#!/usr/bin/env bash +# Tests the cert-gate guards: scripts/cert-gate-skip-guard.sh, scripts/cert-gate-zero-test-guard.sh, +# scripts/cert-gate-summary.sh, and their wiring in scripts/run-cert-gate.sh. +# +# WHY. cert-gate is a REQUIRED check, and `dotnet test` exits 0 both when its filter matches nothing +# and when tests are SKIPPED. The zero-test guard floors the TRX `total` against --list-tests, and +# both of those count a skipped test, so a [Fact(Skip = "...")] on a merge-blocking test used to +# leave cert-gate green (measured 2026-09-30: exit 0 with total=1288 executed=1287). The skip guard +# pins `total - executed` to scripts/cert-gate-skipped.baseline in both directions. A guard is only +# evidence once each of its branches has been seen firing, so this drives the REAL scripts - copied +# into a sandbox repo root, not restated - through fixture TRX files shaped like the real one +# (including its `notExecuted="0"` beside a skipped result), with a stand-in `dotnet` on PATH that +# exits 0 on a skip exactly as the real one does. +# +# Run: bash tests/scripts/cert-gate-guards.test.sh +# Pure bash: no network, no dotnet, no container, no python. + +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +# Bump when you add an assertion; the check at the bottom says why. +EXPECTED_ASSERTIONS=29 + +PASS=0 +FAIL=0 +ok() { PASS=$((PASS + 1)); echo " ok - $1"; } +bad() { FAIL=$((FAIL + 1)); echo " FAIL - $1"; echo " $2"; } + +TMP="$(mktemp -d)" +trap 'rm -rf "${TMP}"' EXIT + +SANDBOX="${TMP}/repo" +BIN="${TMP}/bin" +OUT="${TMP}/out.txt" +SUMMARY_FILE="${TMP}/summary.md" +mkdir -p "${SANDBOX}/scripts" "${BIN}" + +UNDER_TEST=( + run-cert-gate.sh + cert-gate-config.sh + cert-gate-zero-test-guard.sh + cert-gate-skip-guard.sh + cert-gate-summary.sh + cert-gate-skipped.baseline +) + +finish() { + echo + echo "passed: ${PASS} failed: ${FAIL}" + local ran=$((PASS + FAIL)) + if [[ "${ran}" -ne "${EXPECTED_ASSERTIONS}" ]]; then + echo "FAIL - ran ${ran} assertions, expected ${EXPECTED_ASSERTIONS}." + echo " Either this file stopped early or assertions were added without bumping" + echo " EXPECTED_ASSERTIONS. A partial run is not a pass." + exit 1 + fi + [[ "${FAIL}" -eq 0 ]] || exit 1 + exit 0 +} + +# --------------------------------------------------------------------------------------------- +# Non-vacuity controls. Without the real files every case below would run nothing, and the +# "must fail" cases would pass on "file not found". +# --------------------------------------------------------------------------------------------- +echo "== controls ==" +missing=() +for f in "${UNDER_TEST[@]}"; do + if [[ -f "${ROOT}/scripts/${f}" ]]; then + cp "${ROOT}/scripts/${f}" "${SANDBOX}/scripts/${f}" + else + missing+=("scripts/${f}") + fi +done +if [[ "${#missing[@]}" -ne 0 ]]; then + bad "the cert-gate scripts and baseline under test exist" "missing: ${missing[*]}" + echo "Cannot test files that are not there; stopping." + exit 1 +fi +ok "the cert-gate scripts and baseline under test exist and are copied into a sandbox root" + +# Stand-in for dotnet. restore/build succeed; `test --list-tests` lists FAKE_LIST_COUNT tests; any +# other `test` copies FAKE_TRX (when set) to <--results-directory>/cert-gate.trx and exits +# FAKE_TEST_EXIT (default 0 - what the real dotnet test returns when tests are only skipped). +cat > "${BIN}/dotnet" <<'FAKE' +#!/usr/bin/env bash +case "${1:-}" in + restore|build) exit 0 ;; + test) + for a in "$@"; do + if [[ "${a}" == "--list-tests" ]]; then + echo "The following Tests are available:" + for ((i = 1; i <= ${FAKE_LIST_COUNT:-0}; i++)); do echo " Ashlar.Tests.Fake.Listed.Test${i}"; done + exit 0 + fi + done + dir="" prev="" + for a in "$@"; do + if [[ "${prev}" == "--results-directory" ]]; then dir="${a}"; fi + prev="${a}" + done + if [[ -n "${FAKE_TRX:-}" && -n "${dir}" ]]; then cp "${FAKE_TRX}" "${dir}/cert-gate.trx"; fi + exit "${FAKE_TEST_EXIT:-0}" + ;; +esac +echo "fake dotnet: unexpected invocation: $*" >&2 +exit 97 +FAKE +chmod +x "${BIN}/dotnet" + +# sandbox : run with the fake dotnet first on PATH and the step +# summary sent to a scratch file (never the real one when this runs inside a CI job). Output goes +# to ${OUT}; the exit code is returned. +sandbox() { + env PATH="${BIN}:${PATH}" GITHUB_STEP_SUMMARY="${SUMMARY_FILE}" "$@" > "${OUT}" 2>&1 +} + +resolved="$(env PATH="${BIN}:${PATH}" bash -c 'command -v dotnet')" +if [[ "${resolved}" == "${BIN}/dotnet" ]]; then + ok "the sandbox resolves dotnet to the stand-in, not a real SDK" +else + bad "the sandbox resolves dotnet to the stand-in, not a real SDK" "resolved: ${resolved:-}" + echo "Every case below would run a real dotnet; stopping." + finish +fi + +# make_trx [] +# Shaped like the TRX a real cert-gate run writes: a UTF-8 BOM, results BEFORE the summary, a +# skipped result as outcome="NotExecuted", and - as measured on 2026-09-30 - executed excluding the +# skip while notExecuted still reads 0. +make_trx() { + local file="$1" passed="$2" skipped="$3" noise="${4:-}" + local total=$((passed + skipped)) i + { + printf '\xEF\xBB\xBF\n' + echo '' + echo ' ' + for ((i = 1; i <= passed; i++)); do + echo " " + if [[ "${i}" -eq 1 && -n "${noise}" ]]; then + echo " ${noise}" + fi + echo " " + done + for ((i = 1; i <= skipped; i++)); do + echo " " + echo " mutation: cert-skip audit" + echo " " + done + echo ' ' + echo ' ' + echo " " + echo ' ' + echo '' + } > "${file}" +} + +set_baseline() { printf '%b' "$1" > "${SANDBOX}/scripts/cert-gate-skipped.baseline"; } +restore_committed_baseline() { cp "${ROOT}/scripts/cert-gate-skipped.baseline" "${SANDBOX}/scripts/cert-gate-skipped.baseline"; } + +SKIP_GUARD="${SANDBOX}/scripts/cert-gate-skip-guard.sh" +ZERO_GUARD="${SANDBOX}/scripts/cert-gate-zero-test-guard.sh" +SUMMARY_SH="${SANDBOX}/scripts/cert-gate-summary.sh" +RUN_GATE="${SANDBOX}/scripts/run-cert-gate.sh" + +# expect_rc