diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 56bf80a9bcfc..8a046c7d8d7e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -189,7 +189,7 @@ vercel.json @getsentry/owners-js-bu /.github/file-filters.yml @getsentry/owners-js-build build-utils/ @getsentry/owners-js-build oxlint.config.ts @getsentry/owners-js-build -oxlint-suppressions.json @getsentry/owners-js-build +oxlint-suppressions.json @getsentry/design-engineering /scripts/custom-oxlint.ts @getsentry/owners-js-build jest-balance.json @getsentry/owners-js-build jest.config.ts @getsentry/owners-js-build diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index f25e9dd7e445..6e29ce22ee45 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -92,6 +92,7 @@ jobs: core.setOutput('base', await findVerifiedLintBase({github, context, core})); - name: Verify lint ratchet + id: ratchet env: EVENT_NAME: ${{ github.event_name }} SENTRY_OXLINT_VERIFIED_BASE: ${{ steps.lint-baseline.outputs.base }} @@ -102,6 +103,13 @@ jobs: pnpm run lint:js --ci fi + # The ratchet passes with stale budgets inherited from earlier merges, but only + # exact budgets equal live debt. find-verified-lint-base.js reuses master budgets + # only after this step succeeds. + - name: Verify exact lint budgets + if: steps.ratchet.outputs.budgets == 'exact' + run: echo 'Committed oxlint budgets match live debt.' + # Decide whether to lint just the PR's impacted files or everything (see # frontend-changed-scope.sh): we scope to impacted files only when a merge base # resolves and changes are under static/ or to oxlint-suppressions.json. diff --git a/.github/workflows/scripts/find-verified-lint-base.js b/.github/workflows/scripts/find-verified-lint-base.js index a766690fdfe4..d09651912b79 100644 --- a/.github/workflows/scripts/find-verified-lint-base.js +++ b/.github/workflows/scripts/find-verified-lint-base.js @@ -1,5 +1,9 @@ import {execFileSync} from 'node:child_process'; +// A passing ratchet can carry stale budgets above live debt, so only this step proves +// that committed budgets can stand in for scanning the base. +export const EXACT_BUDGETS_STEP = 'Verify exact lint budgets'; + export async function findVerifiedLintBase({github, context, core}) { const git = args => execFileSync('git', args, { @@ -51,7 +55,7 @@ export async function findVerifiedLintBase({github, context, core}) { job.name === 'oxlint' && job.head_sha === run.head_sha && job.steps?.some( - step => step.name === 'Verify lint ratchet' && step.conclusion === 'success' + step => step.name === EXACT_BUDGETS_STEP && step.conclusion === 'success' ) ) ) { diff --git a/.github/workflows/scripts/find-verified-lint-base.test.js b/.github/workflows/scripts/find-verified-lint-base.test.js index 986e3ffb3712..e5a2a5561b3e 100644 --- a/.github/workflows/scripts/find-verified-lint-base.test.js +++ b/.github/workflows/scripts/find-verified-lint-base.test.js @@ -5,7 +5,7 @@ import {tmpdir} from 'node:os'; import path from 'node:path'; import {test} from 'node:test'; -import {findVerifiedLintBase} from './find-verified-lint-base.js'; +import {EXACT_BUDGETS_STEP, findVerifiedLintBase} from './find-verified-lint-base.js'; function fixture(t, changed = 'src/example.py') { const original = process.cwd(); @@ -41,7 +41,10 @@ function fixture(t, changed = 'src/example.py') { const job = { name: 'oxlint', head_sha: verified, - steps: [{name: 'Verify lint ratchet', conclusion: 'success'}], + steps: [ + {name: 'Verify lint ratchet', conclusion: 'success'}, + {name: EXACT_BUDGETS_STEP, conclusion: 'success'}, + ], }; const runs = [run]; const jobs = [job]; @@ -122,12 +125,12 @@ test('changed source, assets, configuration, or budgets invalidate ancestor proo test('a skipped, failed, missing, or unrelated verification cannot certify the base', async t => { const {run, job, find} = fixture(t); for (const conclusion of ['skipped', 'failure', null]) { - job.steps[0].conclusion = conclusion; + job.steps[1].conclusion = conclusion; assert.equal(await find(), ''); } job.steps = [{name: 'oxlint (all files)', conclusion: 'success'}]; assert.equal(await find(), ''); - job.steps = [{name: 'Verify lint ratchet', conclusion: 'success'}]; + job.steps = [{name: EXACT_BUDGETS_STEP, conclusion: 'success'}]; job.head_sha = '0'.repeat(40); assert.equal(await find(), ''); job.head_sha = run.head_sha; @@ -138,6 +141,14 @@ test('a skipped, failed, missing, or unrelated verification cannot certify the b assert.equal(await find(), ''); }); +test('a passing ratchet with stale budgets cannot certify the base', async t => { + const {job, find} = fixture(t); + job.steps[1].conclusion = 'skipped'; + assert.equal(await find(), ''); + job.steps = [{name: 'Verify lint ratchet', conclusion: 'success'}]; + assert.equal(await find(), ''); +}); + test('renaming a lint input to a backend path invalidates ancestor proof', async t => { const {git, base, write, commit, find} = fixture(t); git('checkout', '--quiet', '--detach', base); diff --git a/scripts/custom-oxlint.ts b/scripts/custom-oxlint.ts index 52eca002ce61..dc72492843d8 100644 --- a/scripts/custom-oxlint.ts +++ b/scripts/custom-oxlint.ts @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import {execFileSync, spawn} from 'node:child_process'; import {randomUUID} from 'node:crypto'; import { + appendFileSync, cpSync, existsSync, linkSync, @@ -375,12 +376,70 @@ function transferRenames(base: string, counts: Suppressions) { return counts; } -async function ciBaseline(base: string, allowed: Set, policy: string) { - const changes = [ +function changedFiles(base: string) { + return [ ...git(root, ['diff', '--name-only', '--no-renames', '-z', base, '--']).split('\0'), ...git(root, ['ls-files', '--others', '--exclude-standard', '-z']).split('\0'), - ]; - const policyChanged = changes.some( + ].filter(Boolean); +} + +function committedBudgets(base: string, allowed?: Set) { + if (!git(root, ['ls-tree', '--name-only', base, '--', 'oxlint-suppressions.json'])) { + return; + } + return parseSuppressions( + git(root, ['show', `${base}:oxlint-suppressions.json`]), + allowed + ); +} + +// Merges that are not up to date with master can each prune budgets independently, +// so master can carry budgets above live debt. That inherited slack is not new debt. +// A change must prune only budgets it edited and reductions in live debt since the +// base, including reductions caused indirectly by edits to other files. +function staleBudgets( + committed: Suppressions, + counts: Suppressions, + base?: {counts: Suppressions; ref: string} +) { + const stale: Array<{count: number; file: string; rule: string}> = []; + for (const [file, rules] of Object.entries(committed)) { + for (const [rule, {count}] of Object.entries(rules)) { + if (count > (counts[file]?.[rule]?.count ?? 0)) { + stale.push({file, rule, count}); + } + } + } + const live = (file: string, rule: string) => counts[file]?.[rule]?.count ?? 0; + const baseBudgets = base && stale.length > 0 ? committedBudgets(base.ref) : undefined; + if (base && baseBudgets) { + transferRenames(base.ref, baseBudgets); + } + const owned = stale.filter( + ({file, rule, count}) => + base !== undefined && + (baseBudgets?.[file]?.[rule]?.count !== count || + live(file, rule) < (base.counts[file]?.[rule]?.count ?? 0)) + ); + const describe = ({file, rule, count}: (typeof stale)[number]) => + `${file} ${rule}: budget ${count}, ${live(file, rule)} violations`; + return { + owned: owned.map(describe), + inherited: stale.filter(entry => !owned.includes(entry)).map(describe), + }; +} + +function warn(title: string, details: string[]) { + console.warn([title, ...details].join('\n')); + if (process.env.GITHUB_ACTIONS === 'true') { + const escape = (value: string) => + value.replaceAll('%', '%25').replaceAll('\r', '%0D').replaceAll('\n', '%0A'); + console.log(`::warning title=${escape(title)}::${escape(details.join('\n'))}`); + } +} + +async function ciBaseline(base: string, allowed: Set, policy: string) { + const policyChanged = changedFiles(base).some( file => [ 'oxlint.config.ts', @@ -394,17 +453,11 @@ async function ciBaseline(base: string, allowed: Set, policy: string) { file ) ); - if ( - process.env.SENTRY_OXLINT_VERIFIED_BASE !== base || - policyChanged || - !git(root, ['ls-tree', '--name-only', base, '--', 'oxlint-suppressions.json']) - ) { - return baseScan(base, allowed, policy); - } - return transferRenames( - base, - parseSuppressions(git(root, ['show', `${base}:oxlint-suppressions.json`]), allowed) - ); + const budgets = + process.env.SENTRY_OXLINT_VERIFIED_BASE === base && !policyChanged + ? committedBudgets(base, allowed) + : undefined; + return budgets ? transferRenames(base, budgets) : baseScan(base, allowed, policy); } async function baseScan(base: string, allowed: Set, policy: string) { @@ -512,6 +565,8 @@ Put a maintenance flag first. Maintenance always scans all files. Defaults to the merge base of HEAD and origin/master. --ci [--base REF] Verify committed budgets match live debt and fit REF. Reuse verified REF budgets unless lint policy changed. + Stale budgets fail only for edited budgets or debt + reduced since REF, and only warn without --base. --enroll --base REF Enroll rules using trusted source at REF. --prune Reduce remaining budgets with a full type-aware scan. --backlog [--rule RULE] [--file PATH] [--json] @@ -708,29 +763,48 @@ Native oxlint options: fits(current.counts, reduced, current.findings); replacement = reduced; } else { + let base: {counts: Suppressions; ref: string} | undefined; if (command !== 'ci' || values.base) { let ref = values.base; if (!ref) { ref = git(root, ['merge-base', 'HEAD', revision('origin/master')]); } - const base = revision(ref); - fits( - current.counts, - await (command === 'ci' - ? ciBaseline(base, allowed, policy) - : baseScan(base, allowed, policy)), - current.findings - ); + ref = revision(ref); + base = { + ref, + counts: await (command === 'ci' + ? ciBaseline(ref, allowed, policy) + : baseScan(ref, allowed, policy)), + }; + fits(current.counts, base.counts, current.findings); } if (command === 'enroll') { replacement = current.counts; } else if (command === 'ci') { assert(committed, 'The suppression asset must be committed'); fits(current.counts, committed, current.findings); - if (serialize(committed) !== serialize(current.counts)) { + const {owned, inherited} = staleBudgets(committed, current.counts, base); + if (owned.length > 0) { process.exitCode = 1; throw new Error( - 'Suppression budgets are stale. Run pnpm run lint:js --prune and commit the updated oxlint-suppressions.json.' + [ + 'Suppression budgets are stale. Run pnpm run lint:js --prune and commit the updated oxlint-suppressions.json.', + ...owned, + ].join('\n') + ); + } + if (inherited.length > 0) { + warn( + 'Ignoring stale suppression budgets inherited from the base. Prune them separately with pnpm run lint:js --prune.', + inherited + ); + } + // Only exact budgets equal live debt, so only they may be reused as a verified + // baseline by find-verified-lint-base.js. + if (process.env.GITHUB_OUTPUT) { + appendFileSync( + process.env.GITHUB_OUTPUT, + `budgets=${inherited.length > 0 ? 'stale' : 'exact'}\n` ); } } diff --git a/scripts/customOxlint.test.ts b/scripts/customOxlint.test.ts index a3340ebbbe22..9d1cd3d7fa32 100644 --- a/scripts/customOxlint.test.ts +++ b/scripts/customOxlint.test.ts @@ -37,7 +37,7 @@ function fixture(t: {after: (cleanup: () => void) => void}) { ); const git = (...args: string[]) => execFileSync('git', ['-C', directory, ...args], {encoding: 'utf8'}).trim(); - const env = {...process.env, SENTRY_OXLINT_VERIFIED_BASE: ''}; + const env = {...process.env, GITHUB_OUTPUT: '', SENTRY_OXLINT_VERIFIED_BASE: ''}; const lint = (...args: string[]) => spawnSync(process.execPath, [runner, ...args], { cwd: directory, @@ -143,6 +143,28 @@ test('lint-only paths do not count as frontend changes', t => { assert(matches('frontend_all').includes('package.json')); }); +// Workflow script tests run without installed dependencies, so the YAML contract lives here. +test('verified lint bases require the exact budgets step after the ratchet', () => { + const step = 'Verify exact lint budgets'; + const {jobs} = parse( + readFileSync(path.join(root, '.github/workflows/frontend.yml'), 'utf8') + ); + const steps: Array<{id?: string; if?: string; name?: string; run?: string}> = + jobs.oxlint.steps; + const ratchet = steps.findIndex(({name}) => name === 'Verify lint ratchet'); + const exact = steps.findIndex(({name}) => name === step); + assert(ratchet >= 0 && exact > ratchet); + assert.equal(steps[ratchet]!.id, 'ratchet'); + assert.match(steps[ratchet]!.run ?? '', /pnpm run lint:js --ci/); + assert.equal(steps[exact]!.if, "steps.ratchet.outputs.budgets == 'exact'"); + assert( + readFileSync( + path.join(root, '.github/workflows/scripts/find-verified-lint-base.js'), + 'utf8' + ).includes(`EXACT_BUDGETS_STEP = '${step}'`) + ); +}); + test('override-only rules are enrolled with editor warnings and scoped CLI errors', t => { const {directory, write, lint} = fixture(t); const registry = `{ @@ -315,11 +337,95 @@ test('CI scans head once and rejects increased or stale budgets', t => { assert.match(stale.stderr, /Suppression budgets are stale/); assert.match(stale.stderr, /pnpm run lint:js --prune/); assert.match(stale.stderr, /commit the updated oxlint-suppressions.json/); + assert.match(stale.stderr, /source.js no-debugger: budget 2, 0 violations/); assert.doesNotMatch(stale.stderr, /fix:oxlint|--enroll|New incubator violations/); write('oxlint-suppressions.json', '{}'); assert.equal(ci(1).status, 0); }); +test('CI only warns about stale budgets inherited from the base', t => { + const {directory, write, commit, lint, ci, env} = ciFixture(t); + const output = '.artifacts/github-output'; + const budgets = () => readFileSync(path.join(directory, output), 'utf8').trim(); + env.GITHUB_OUTPUT = path.join(directory, output); + write('source.js', ''); + commit(); + write(output, ''); + const master = lint('--ci'); + assert.equal(master.status, 0, master.stderr); + assert.match(master.stderr, /Ignoring stale suppression budgets/); + assert.match(master.stderr, /source.js no-debugger: budget 1, 0 violations/); + assert.match(master.stdout, /Incubator ratchet passed/); + assert.equal(budgets(), 'budgets=stale'); + // A stale base cannot be verified, so each PR check scans the base. + write('source.js', 'void 0;\n'); + write(output, ''); + const unrelated = ci(2, ''); + assert.equal(unrelated.status, 0, unrelated.stderr); + assert.match(unrelated.stderr, /source.js no-debugger: budget 1, 0 violations/); + assert.equal(budgets(), 'budgets=stale'); + write('source.js', 'debugger;\n'); + const refill = ci(2, ''); + assert.equal(refill.status, 1, refill.stderr); + assert.match(refill.stderr, /source.js no-debugger: 1 violations, budget 0/); + write('source.js', ''); + write( + 'oxlint-suppressions.json', + JSON.stringify({'source.js': {'no-debugger': {count: 2}}}) + ); + const edited = ci(2, ''); + assert.equal(edited.status, 1, edited.stderr); + assert.match(edited.stderr, /Suppression budgets are stale/); + assert.match(edited.stderr, /source.js no-debugger: budget 2, 0 violations/); + write('oxlint-suppressions.json', '{}'); + write(output, ''); + const pruned = ci(2, ''); + assert.equal(pruned.status, 0, pruned.stderr); + assert.doesNotMatch(pruned.stderr, /stale/); + assert.equal(budgets(), 'budgets=exact'); +}); + +test('CI requires pruning debt reduced indirectly by another file', t => { + const {write, lint, commit, env} = fixture(t); + write('pnpm-workspace.yaml', 'packages: []'); + write('types.d.ts', 'declare function run(): Promise;\n'); + write('index.ts', 'run();\n'); + write( + 'tsconfig.json', + JSON.stringify({ + compilerOptions: {strict: true, target: 'ESNext', module: 'NodeNext'}, + include: ['*.ts'], + }) + ); + write( + 'oxlint.config.ts', + ` + export const incubator = {rules: {'typescript/no-floating-promises': 'error'}}; + export default { + plugins: ['typescript'], + categories: {correctness: 'off'}, + options: {typeAware: true}, + ...incubator, + }; + ` + ); + write( + 'oxlint-suppressions.json', + JSON.stringify({'index.ts': {'typescript/no-floating-promises': {count: 1}}}) + ); + commit(); + env.SENTRY_OXLINT_VERIFIED_BASE = ''; + assert.equal(lint('--ci', '--base', 'HEAD').status, 0); + write('types.d.ts', 'declare function run(): void;\n'); + const reduced = lint('--ci', '--base', 'HEAD'); + assert.equal(reduced.status, 1, reduced.stdout + reduced.stderr); + assert.match(reduced.stderr, /Suppression budgets are stale/); + assert.match( + reduced.stderr, + /index.ts typescript\/no-floating-promises: budget 1, 0 violations/ + ); +}); + test('CI and prune report violations exceeding committed budgets', t => { const {directory, write, lint, ci} = ciFixture(t); write('oxlint-suppressions.json', '{}');