diff --git a/.changeset/long-radios-join.md b/.changeset/long-radios-join.md new file mode 100644 index 000000000..e7140b780 --- /dev/null +++ b/.changeset/long-radios-join.md @@ -0,0 +1,5 @@ +--- +"@systemfsoftware/stryker-config": none +--- + +sharedConfig sets disableBail and mutator.mutantSetPolicy 'full' when STRYKER_KILL_MATRIX=1, for the kill-matrix CI lane; unset, the config is unchanged. diff --git a/.github/workflows/kill-matrix.yml b/.github/workflows/kill-matrix.yml new file mode 100644 index 000000000..eacc59dd2 --- /dev/null +++ b/.github/workflows/kill-matrix.yml @@ -0,0 +1,288 @@ +name: Kill matrix + +on: + workflow_call: + inputs: + projects: + description: Comma-separated projects to mutate, the dogfood corpus. + type: string + default: packages/stryker-js,packages/stryker-js-typescript-checker,packages/stryker-js-vitest-runner,test/e2e-core + workflow_dispatch: + inputs: + projects: + description: Comma-separated projects to mutate, the dogfood corpus. + type: string + default: packages/stryker-js,packages/stryker-js-typescript-checker,packages/stryker-js-vitest-runner,test/e2e-core + +concurrency: + group: kill-matrix-${{ github.ref }} + cancel-in-progress: false + +permissions: + contents: read + +env: + TARGET_SECONDS: 200 + MAX_SHARDS: 20 + PROJECTS: ${{ inputs.projects }} + PUBLISHED_CLI: ${{ github.workspace }}/packages/stryker-js/node_modules/.bin/stryker + PLAN: plan.json + KILL_MATRIX_OUT: kill-matrix-out + KILL_MATRIX_ARTIFACT: kill-matrix-${{ github.run_number }} + STRYKER_KILL_MATRIX: "1" + +jobs: + plan: + name: kill-matrix · plan + runs-on: ubuntu-latest + timeout-minutes: 20 + outputs: + matrix: ${{ steps.plan.outputs.matrix }} + count: ${{ steps.plan.outputs.count }} + steps: + - uses: actions/checkout@v7 + - uses: pnpm/action-setup@v6 + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: pnpm + - uses: ./.github/actions/released-tarballs + - run: pnpm install --frozen-lockfile + + - id: plan + name: Plan every mutant, from scratch + run: | + $PUBLISHED_CLI plan \ + --target-seconds "$TARGET_SECONDS" \ + --max-shards "$MAX_SHARDS" \ + --projects "$PROJECTS" \ + --full \ + --out "$PLAN" + echo "matrix=$(jq -c '.matrix' "$PLAN")" >> "$GITHUB_OUTPUT" + echo "count=$(jq '.matrix.include | length' "$PLAN")" >> "$GITHUB_OUTPUT" + + - uses: actions/upload-artifact@v7 + with: + name: kill-matrix-plan + path: ${{ env.PLAN }} + retention-days: 7 + + build: + name: kill-matrix · build + runs-on: ubuntu-latest + timeout-minutes: 20 + outputs: + key: ${{ steps.key.outputs.key }} + steps: + - uses: actions/checkout@v7 + + - id: key + name: Content key of the build + env: + KEY: mutation-build-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml', 'turbo.json', 'packages/**', 'test/e2e-core/**') }} + run: echo "key=$KEY" >> "$GITHUB_OUTPUT" + + - uses: pnpm/action-setup@v6 + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: pnpm + - uses: ./.github/actions/released-tarballs + - run: pnpm install --frozen-lockfile + + - name: Restore the build the mutation lane cached + uses: actions/cache/restore@v6 + with: + path: .turbo/cache + key: ${{ steps.key.outputs.key }} + restore-keys: | + mutation-build-${{ runner.os }}- + + - name: Build every mutated project + run: | + mapfile -t filters < <(tr ',' '\n' <<< "$PROJECTS" | sed 's|^|--filter=./|') + pnpm exec turbo run build --output-logs=errors-only --log-order=stream "${filters[@]}" + + mutation: + name: kill-matrix · shard ${{ matrix.shard }} + needs: [plan, build] + if: ${{ needs.plan.result == 'success' && needs.build.result == 'success' && needs.plan.outputs.count != '0' }} + runs-on: ubuntu-latest + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} + steps: + - uses: actions/checkout@v7 + - uses: pnpm/action-setup@v6 + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: pnpm + - uses: ./.github/actions/released-tarballs + - run: pnpm install --frozen-lockfile + - uses: actions/download-artifact@v8 + with: + name: kill-matrix-plan + + - name: Restore the build every shard shares + uses: actions/cache/restore@v6 + with: + path: .turbo/cache + key: ${{ needs.build.outputs.key }} + + - name: Build the projects this shard mutates + env: + SHARD: ${{ matrix.shard }} + run: | + mapfile -t filters < <( + jq -r --arg shard "$SHARD" \ + '.shards[] | select("\(.index)/\(.count)" == $shard) | .projects[].project' "$PLAN" \ + | sort -u | sed 's|^|--filter=./|' + ) + pnpm exec turbo run build --output-logs=errors-only --log-order=stream "${filters[@]}" + + - id: shard + name: Name the shard output + env: + SHARD: ${{ matrix.shard }} + run: echo "slug=${SHARD//\//of}" >> "$GITHUB_OUTPUT" + + - name: Mutation, recording every killer + env: + SHARD: ${{ matrix.shard }} + run: | + $PUBLISHED_CLI run \ + --plan "$PLAN" \ + --shard "$SHARD" \ + --out "reports/shards/${{ steps.shard.outputs.slug }}" + + - id: upload + name: Upload the shard's incremental parts + if: always() + uses: actions/upload-artifact@v7 + with: + name: kill-matrix-shard-${{ steps.shard.outputs.slug }} + path: reports/shards + if-no-files-found: error + retention-days: 7 + overwrite: true + + - name: Annotate a failed shard + if: ${{ failure() }} + env: + SHARD: ${{ matrix.shard }} + run: | + echo "::error title=Kill matrix shard $SHARD failed::The full re-verification of shard $SHARD did not finish, so the kill matrix is incomplete. Read the log at ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} and any partial parts in artifact kill-matrix-shard-${SHARD//\//of}." + + collect: + name: kill-matrix · collect + needs: [plan, mutation] + if: ${{ needs.mutation.result == 'success' && needs.plan.result == 'success' && needs.plan.outputs.count != '0' }} + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + - uses: pnpm/action-setup@v6 + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: pnpm + - uses: ./.github/actions/released-tarballs + - run: pnpm install --frozen-lockfile + - uses: actions/download-artifact@v8 + with: + name: kill-matrix-plan + - name: Download every shard's incremental parts into shards/of + uses: actions/download-artifact@v8 + with: + pattern: kill-matrix-shard-* + path: shards + merge-multiple: true + + - name: Merge into one incremental report per project + run: | + shards=() + while read -r shard; do + shards+=("shards/${shard//\//of}") + done < <(jq -r '.shards[] | "\(.index)/\(.count)"' "$PLAN") + $PUBLISHED_CLI merge \ + --plan "$PLAN" \ + "${shards[@]}" \ + --out reports/mutation + for dir in ${PROJECTS//,/ }; do + merged="reports/mutation/$dir/stryker-incremental.json" + if [ ! -f "$merged" ]; then + echo "::error title=No kill matrix for $dir::$merged is missing, so this run did not cover $dir" + exit 1 + fi + mkdir -p "$KILL_MATRIX_OUT/$dir" + cp "$merged" "$KILL_MATRIX_OUT/$dir/stryker-incremental.json" + done + + - name: Summarise the kill matrix + run: | + node --input-type=module - <<'NODE' + import { appendFileSync, readFileSync } from 'node:fs' + + const planned = new Map() + for (const shard of JSON.parse(readFileSync(process.env.PLAN, 'utf8')).shards) { + for (const { project, mutants } of shard.projects) { + planned.set(project, [...(planned.get(project) ?? []), ...mutants]) + } + } + const rows = [] + const notFull = [] + const unreported = [] + let corpusKilled = 0 + let corpusMulti = 0 + for (const project of process.env.PROJECTS.split(',').map((p) => p.trim()).filter(Boolean)) { + const report = JSON.parse(readFileSync(`${process.env.KILL_MATRIX_OUT}/${project}/stryker-incremental.json`, 'utf8')) + if (report.mutantSetPolicy !== 'full') notFull.push(`${project} (${report.mutantSetPolicy})`) + const mutants = Object.values(report.files ?? {}).flatMap((file) => file.mutants ?? []) + const reported = new Set(mutants.map((mutant) => mutant.id)) + const missing = (planned.get(project) ?? []).filter((id) => !reported.has(id)) + if (missing.length > 0) unreported.push(`${project}: ${missing.length} planned ids missing, e.g. ${missing.slice(0, 5).join(' ')}`) + const killed = mutants.filter((mutant) => mutant.status === 'Killed') + const multi = killed.filter((mutant) => (mutant.killedBy ?? []).length >= 2).length + const testsCompleted = mutants.reduce((total, mutant) => total + (mutant.testsCompleted ?? 0), 0) + corpusKilled += killed.length + corpusMulti += multi + rows.push(`| ${project} | ${mutants.length} | ${killed.length} | ${multi} | ${testsCompleted} |`) + } + if (notFull.length > 0) { + console.log(`::error title=Kill matrix is not a full matrix::${notFull.join(', ')} did not run with mutator.mutantSetPolicy 'full'; re-run the kill-matrix lane.`) + process.exitCode = 1 + } + if (unreported.length > 0) { + console.log(`::error title=Kill matrix lost planned mutants::${unreported.join('; ')}. The shard runs emitted a different mutant set than the plan; check that every step loaded the config with STRYKER_KILL_MATRIX=1, then re-run the kill-matrix lane.`) + process.exitCode = 1 + } + if (corpusKilled > 0 && corpusMulti === 0) { + console.log(`::error title=Kill matrix recorded only first killers::${corpusKilled} mutants were killed and none names a second killer, so the runner bailed on the first failing test. Check that every project's config keeps disableBail from sharedConfig, then re-run the kill-matrix lane.`) + process.exitCode = 1 + } + appendFileSync(process.env.GITHUB_STEP_SUMMARY, [ + '## Kill matrix', + '', + '| project | mutants | killed | mutants with ≥2 killers | tests completed |', + '| --- | ---: | ---: | ---: | ---: |', + ...rows, + '', + `Artifact: \`${process.env.KILL_MATRIX_ARTIFACT}\` · run: ${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`, + '', + ].join('\n')) + NODE + + - uses: actions/upload-artifact@v7 + with: + name: ${{ env.KILL_MATRIX_ARTIFACT }} + path: ${{ env.KILL_MATRIX_OUT }}/* + if-no-files-found: error + retention-days: 90 + overwrite: true + + - name: Annotate a failed collect + if: ${{ failure() }} + run: | + echo "::error title=Kill matrix was not published::No complete kill-matrix artifact was produced. Read the log at ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} for the missing project or failed merge, then re-run the kill-matrix lane." diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml index d487b55fe..827d085cb 100644 --- a/.github/workflows/mutation.yml +++ b/.github/workflows/mutation.yml @@ -13,6 +13,13 @@ on: description: Re-verify every mutant instead of reusing the incremental cache type: boolean default: false + lane: + description: Which lane to run. `kill-matrix` runs only the full kill-matrix lane. + type: choice + options: + - all + - kill-matrix + default: all schedule: - cron: "17 3 * * *" @@ -39,7 +46,7 @@ env: jobs: plan: name: plan · shards from the incremental cost record - if: github.event_name != 'push' || github.event.head_commit.author.username != 'github-actions[bot]' + if: ${{ github.event.inputs.lane != 'kill-matrix' && (github.event_name != 'push' || github.event.head_commit.author.username != 'github-actions[bot]') }} runs-on: ubuntu-latest timeout-minutes: 20 permissions: @@ -95,7 +102,7 @@ jobs: build: name: build · once for every shard - if: github.event_name != 'push' || github.event.head_commit.author.username != 'github-actions[bot]' + if: ${{ github.event.inputs.lane != 'kill-matrix' && (github.event_name != 'push' || github.event.head_commit.author.username != 'github-actions[bot]') }} runs-on: ubuntu-latest timeout-minutes: 20 outputs: @@ -150,7 +157,7 @@ jobs: mutation: name: mutation · shard ${{ matrix.shard }} needs: [plan, build] - if: ${{ needs.plan.result == 'success' && needs.build.result == 'success' && needs.plan.outputs.count != '0' }} + if: ${{ github.event.inputs.lane != 'kill-matrix' && needs.plan.result == 'success' && needs.build.result == 'success' && needs.plan.outputs.count != '0' }} runs-on: ubuntu-latest timeout-minutes: 60 strategy: @@ -250,7 +257,7 @@ jobs: report: name: Mutation report needs: [plan, mutation] - if: ${{ !cancelled() && needs.plan.result == 'success' && needs.plan.outputs.count != '0' }} + if: ${{ github.event.inputs.lane != 'kill-matrix' && !cancelled() && needs.plan.result == 'success' && needs.plan.outputs.count != '0' }} runs-on: ubuntu-latest timeout-minutes: 30 permissions: @@ -350,3 +357,10 @@ jobs: gh api --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts" \ --jq '.artifacts[] | select(.name | startswith("mutation-shard-")) | .id' \ | while read -r id; do gh api -X DELETE "repos/$GITHUB_REPOSITORY/actions/artifacts/$id"; done + + kill-matrix: + name: Kill matrix + if: ${{ (github.event_name == 'push' && github.event.head_commit.author.username != 'github-actions[bot]') || github.event.inputs.lane == 'kill-matrix' }} + permissions: + contents: read + uses: ./.github/workflows/kill-matrix.yml diff --git a/packages/toolchain/stryker-config/lib/base.d.ts b/packages/toolchain/stryker-config/lib/base.d.ts index 65be7062d..244ad0442 100644 --- a/packages/toolchain/stryker-config/lib/base.d.ts +++ b/packages/toolchain/stryker-config/lib/base.d.ts @@ -12,6 +12,8 @@ export interface SharedConfig { cleanTempDir: 'always' thresholds: { high: number; low: number; break: number } concurrency?: string + disableBail?: boolean + mutator?: { mutantSetPolicy: 'default' | 'full'; [key: string]: unknown } } export const sharedConfig: SharedConfig diff --git a/packages/toolchain/stryker-config/lib/base.js b/packages/toolchain/stryker-config/lib/base.js index cff53e075..c5dfbe75a 100644 --- a/packages/toolchain/stryker-config/lib/base.js +++ b/packages/toolchain/stryker-config/lib/base.js @@ -9,6 +9,8 @@ const envConcurrency = process.env['STRYKER_CONCURRENCY'] ?? const restoredShardReports = ['reports/stryker-incremental-*.json'] +const killMatrixLane = process.env['STRYKER_KILL_MATRIX'] === '1' + export const sharedConfig = { packageManager: 'pnpm', reporters: isAgent || isCI ? ['json', 'html'] : ['progress', 'html', 'json'], @@ -22,6 +24,7 @@ export const sharedConfig = { cleanTempDir: 'always', thresholds: { high: 100, low: 80, break: 100 }, ...(envConcurrency !== undefined ? { concurrency: envConcurrency } : {}), + ...(killMatrixLane ? { disableBail: true, mutator: { mutantSetPolicy: 'full' } } : {}), } /**