diff --git a/docs/src/api/class-tracing.md b/docs/src/api/class-tracing.md index 3d46033821be1..906aece2961f6 100644 --- a/docs/src/api/class-tracing.md +++ b/docs/src/api/class-tracing.md @@ -146,6 +146,13 @@ To specify the final trace zip file name, you need to pass `path` option to Whether to capture screenshots during tracing. Screenshots are used to build a timeline preview. +### option: Tracing.start.coverage +* since: v1.64 +* langs: js +- `coverage` <[boolean]> + +Whether to collect code coverage from istanbul-instrumented application code. Build the application with an istanbul instrumentation plugin, for example [`vite-plugin-istanbul`](https://www.npmjs.com/package/vite-plugin-istanbul) or [`babel-plugin-istanbul`](https://www.npmjs.com/package/babel-plugin-istanbul), so that pages expose the `window.__coverage__` object. Playwright collects accumulated counters from all pages and frames, including right before navigations and page closes, and stores them in istanbul format inside the trace file. + ### option: Tracing.start.snapshots * since: v1.12 * langs: js diff --git a/docs/src/test-api/class-testoptions.md b/docs/src/test-api/class-testoptions.md index 5777731f99a99..06cfe0e883f0c 100644 --- a/docs/src/test-api/class-testoptions.md +++ b/docs/src/test-api/class-testoptions.md @@ -673,6 +673,7 @@ export default defineConfig({ - `dom` ?<[boolean]> Capture DOM snapshot on every action and record network activity. Optional. - `aria` ?<[boolean]> Capture aria snapshot of the page on every action. Optional. - `screen` ?<[boolean]> Capture a screenshot of the page on every action. Optional. + - `coverage` ?<[boolean]> Whether to collect coverage from istanbul-instrumented application code into the trace. Defaults to false. Optional. - `sources` ?<[boolean]> Whether to include source files for trace actions. Defaults to true. Optional. Whether to record trace for each test. Defaults to `'off'`. The initial run of a test is the "first run"; subsequent runs caused by [retries](../test-retries.md) are "retries". diff --git a/packages/injected/src/coverageScript.ts b/packages/injected/src/coverageScript.ts new file mode 100644 index 0000000000000..1852ece1602be --- /dev/null +++ b/packages/injected/src/coverageScript.ts @@ -0,0 +1,165 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { kCoverageStashError, kCoverageStashPrefix } from '@isomorphic/istanbulCoverage'; + +import type { IstanbulCoverage, IstanbulCoverageDelta, IstanbulFileCoverageDelta } from '@isomorphic/istanbulCoverage'; + +// Reports `__coverage__` as a delta: reading resets the counters, maps are sent once per file. +export class CoverageScript { + private _global: typeof globalThis; + private _takeName: string; + private _sessionId: string; + private _reportedFiles = new Set(); + private _onPageHide = () => this._stashCurrent(); + + constructor(global: typeof globalThis, takeName: string, sessionId: string) { + this._global = global; + this._takeName = takeName; + this._sessionId = sessionId; + (global as any)[takeName] = Object.assign(() => this.take(), { dispose: () => this.dispose() }); + this._resetCounters(); + // Counters die with the document and unload-time calls are not delivered. + global.addEventListener('pagehide', this._onPageHide); + } + + private _resetCounters() { + const coverage: IstanbulCoverage | undefined = (this._global as any).__coverage__; + for (const fileCoverage of Object.values(coverage || {})) { + takeCounters(fileCoverage.s); + takeCounters(fileCoverage.f); + takeBranchCounters(fileCoverage.b); + } + } + + dispose() { + this._global.removeEventListener('pagehide', this._onPageHide); + delete (this._global as any)[this._takeName]; + } + + take(): string[] { + const chunks = takeCoverageStashes(this._global, this._sessionId); + const delta = this._takeCurrent(); + if (delta) + chunks.push(JSON.stringify({ data: delta })); + return chunks; + } + + private _takeCurrent(): IstanbulCoverageDelta | undefined { + const coverage: IstanbulCoverage | undefined = (this._global as any).__coverage__; + if (!coverage) + return undefined; + const delta: IstanbulCoverageDelta = {}; + for (const [file, fileCoverage] of Object.entries(coverage)) { + const s = takeCounters(fileCoverage.s); + const f = takeCounters(fileCoverage.f); + const b = takeBranchCounters(fileCoverage.b); + // Every file is reported once, so that the never hit ones are accounted for. + const isFirstReport = !this._reportedFiles.has(file); + if (!isFirstReport && !s && !f && !b) + continue; + const entry: IstanbulFileCoverageDelta = { path: fileCoverage.path, s: s || {}, f: f || {}, b: b || {} }; + if (isFirstReport) { + entry.statementMap = fileCoverage.statementMap; + entry.fnMap = fileCoverage.fnMap; + entry.branchMap = fileCoverage.branchMap; + this._reportedFiles.add(file); + } + delta[file] = entry; + } + return Object.keys(delta).length ? delta : undefined; + } + + private _stashCurrent() { + const delta = this._takeCurrent(); + if (!delta) + return; + // Several documents can pick up the same stash, the id discards the copies. + const id = Math.random().toString(36).slice(2); + try { + this._global.localStorage.setItem(kCoverageStashPrefix + this._sessionId + '.' + id, JSON.stringify({ id, data: delta })); + } catch (error) { + // The counters are lost, the next take surfaces the error. + try { + this._global.localStorage.setItem(stashErrorKey(this._sessionId), String(error)); + } catch { + } + } + } +} + +function stashErrorKey(sessionId: string) { + return kCoverageStashPrefix + 'error.' + sessionId; +} + +// Stashes of other sessions are stale, e.g. left by a previous run in a persistent profile. +export function takeCoverageStashes(global: typeof globalThis, sessionId: string): string[] { + const result: string[] = []; + let storage: Storage; + try { + storage = global.localStorage; + } catch { + return result; + } + const error = storage.getItem(stashErrorKey(sessionId)); + if (error) { + storage.removeItem(stashErrorKey(sessionId)); + throw new Error(kCoverageStashError + ': ' + error); + } + const sessionPrefix = kCoverageStashPrefix + sessionId + '.'; + const keys: string[] = []; + for (let i = 0; i < storage.length; i++) { + const key = storage.key(i); + if (key && key.startsWith(kCoverageStashPrefix)) + keys.push(key); + } + for (const key of keys) { + const json = key.startsWith(sessionPrefix) ? storage.getItem(key) : undefined; + storage.removeItem(key); + if (json) + result.push(json); + } + return result; +} + +function takeCounters(counters: { [key: string]: number }): { [key: string]: number } | undefined { + let result: { [key: string]: number } | undefined; + for (const key of Object.keys(counters)) { + const count = counters[key]; + if (!count) + continue; + if (!result) + result = {}; + result[key] = count; + counters[key] = 0; + } + return result; +} + +// Branch counters are positional, so a hit branch is reported with the whole array. +function takeBranchCounters(counters: { [key: string]: number[] }): { [key: string]: number[] } | undefined { + let result: { [key: string]: number[] } | undefined; + for (const key of Object.keys(counters)) { + const counts = counters[key]; + if (!counts.some(Boolean)) + continue; + if (!result) + result = {}; + result[key] = counts.slice(); + counts.fill(0); + } + return result; +} diff --git a/packages/injected/src/storageScript.ts b/packages/injected/src/storageScript.ts index f7e4d3780923f..efb3054cef573 100644 --- a/packages/injected/src/storageScript.ts +++ b/packages/injected/src/storageScript.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { kCoverageStashPrefix } from '@isomorphic/istanbulCoverage'; import { parseEvaluationResultValue, serializeAsCallArgument, typedArrayToBase64 } from '@isomorphic/utilityScriptSerializers'; import type { IndexedDBDatabase, OPFSEntry, OriginStorage, SetOriginStorage } from '@protocol/structs'; @@ -170,7 +171,9 @@ export class StorageScript { } async collect(record: { indexedDB: boolean, opfs: boolean }): Promise { - const localStorage = Object.keys(this._global.localStorage).map(name => ({ name, value: this._global.localStorage.getItem(name)! })); + const localStorage = Object.keys(this._global.localStorage) + .filter(name => !name.startsWith(kCoverageStashPrefix)) + .map(name => ({ name, value: this._global.localStorage.getItem(name)! })); const result: SerializedStorage = { localStorage }; if (record.indexedDB) { try { diff --git a/packages/isomorphic/index.ts b/packages/isomorphic/index.ts index bcd969c0ff4a1..146759e178416 100644 --- a/packages/isomorphic/index.ts +++ b/packages/isomorphic/index.ts @@ -21,6 +21,7 @@ export * from './base64'; export * from './colors'; export * from './headers'; export * from './imageUtils'; +export * from './istanbulCoverage'; export * from './jsonSchema'; export * from './locatorGenerators'; export * from './manualPromise'; diff --git a/packages/isomorphic/istanbulCoverage.ts b/packages/isomorphic/istanbulCoverage.ts new file mode 100644 index 0000000000000..7c0111c6f4371 --- /dev/null +++ b/packages/isomorphic/istanbulCoverage.ts @@ -0,0 +1,101 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// The istanbul coverage format, mirrors the shapes from istanbul-lib-coverage. + +export type IstanbulLocation = { + line: number; + column: number; +}; + +export type IstanbulRange = { + start: IstanbulLocation; + end: IstanbulLocation; +}; + +export type IstanbulFunctionMapping = { + name: string; + decl: IstanbulRange; + loc: IstanbulRange; + line?: number; +}; + +export type IstanbulBranchMapping = { + type: string; + loc: IstanbulRange; + locations: IstanbulRange[]; + line?: number; +}; + +export type IstanbulFileCoverage = { + path: string; + statementMap: { [key: string]: IstanbulRange }; + fnMap: { [key: string]: IstanbulFunctionMapping }; + branchMap: { [key: string]: IstanbulBranchMapping }; + s: { [key: string]: number }; + f: { [key: string]: number }; + b: { [key: string]: number[] }; +}; + +export type IstanbulCoverage = { [file: string]: IstanbulFileCoverage }; + +// The maps are only sent with the first report of each file. +export type IstanbulFileCoverageDelta = Partial & Pick; + +export type IstanbulCoverageDelta = { [file: string]: IstanbulFileCoverageDelta }; + +export const kCoverageStashPrefix = '__pwCoverage.'; +export const kCoverageStashError = 'Failed to stash the coverage'; + +export type IstanbulCoverageChunk = { + data: IstanbulCoverageDelta; + id?: string; +}; + +export function sortedIstanbulCoverage(coverage: Map): IstanbulCoverage { + return Object.fromEntries([...coverage.entries()].sort(([a], [b]) => a.localeCompare(b))); +} + +export function mergeIstanbulCoverage(into: Map, data: IstanbulCoverageDelta) { + for (const [file, fileCov] of Object.entries(data)) { + let existing = into.get(file); + if (!existing) { + existing = { path: fileCov.path, statementMap: {}, fnMap: {}, branchMap: {}, s: {}, f: {}, b: {} }; + into.set(file, existing); + } + // The maps can arrive late if the report that carried them was lost. + if (fileCov.statementMap && !Object.keys(existing.statementMap).length) { + existing.statementMap = fileCov.statementMap; + existing.fnMap = fileCov.fnMap || {}; + existing.branchMap = fileCov.branchMap || {}; + // Reports only carry the counters that were hit, zero fill the rest. + for (const key of Object.keys(existing.statementMap)) + existing.s[key] = existing.s[key] || 0; + for (const key of Object.keys(existing.fnMap)) + existing.f[key] = existing.f[key] || 0; + for (const [key, branch] of Object.entries(existing.branchMap)) + existing.b[key] = existing.b[key] || branch.locations.map(() => 0); + } + for (const key of Object.keys(fileCov.s)) + existing.s[key] = (existing.s[key] || 0) + fileCov.s[key]; + for (const key of Object.keys(fileCov.f)) + existing.f[key] = (existing.f[key] || 0) + fileCov.f[key]; + for (const key of Object.keys(fileCov.b)) { + const branches = existing.b[key] || (existing.b[key] = []); + fileCov.b[key].forEach((count, i) => branches[i] = (branches[i] || 0) + count); + } + } +} diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts index be0790ba1a826..5fcd6b1c96abe 100644 --- a/packages/playwright-client/types/types.d.ts +++ b/packages/playwright-client/types/types.d.ts @@ -23268,6 +23268,16 @@ export interface Tracing { * @param options */ start(options?: { + /** + * Whether to collect code coverage from istanbul-instrumented application code. Build the application with an + * istanbul instrumentation plugin, for example + * [`vite-plugin-istanbul`](https://www.npmjs.com/package/vite-plugin-istanbul) or + * [`babel-plugin-istanbul`](https://www.npmjs.com/package/babel-plugin-istanbul), so that pages expose the + * `window.__coverage__` object. Playwright collects accumulated counters from all pages and frames, including right + * before navigations and page closes, and stores them in istanbul format inside the trace file. + */ + coverage?: boolean; + /** * When enabled, the trace is written to an unarchived file that is updated in real time as actions occur, instead of * caching changes and archiving them into a zip file at the end. This is useful for live trace viewing during test diff --git a/packages/playwright-core/src/client/channels.d.ts b/packages/playwright-core/src/client/channels.d.ts index a645cd61a81e3..6eee244b3afe8 100644 --- a/packages/playwright-core/src/client/channels.d.ts +++ b/packages/playwright-core/src/client/channels.d.ts @@ -5075,6 +5075,7 @@ export type TracingTracingStartParams = { snapshotAria?: boolean, snapshotScreen?: boolean, screencast?: boolean, + coverage?: boolean, live?: boolean, }; export type TracingTracingStartOptions = { @@ -5083,6 +5084,7 @@ export type TracingTracingStartOptions = { snapshotAria?: boolean, snapshotScreen?: boolean, screencast?: boolean, + coverage?: boolean, live?: boolean, }; export type TracingTracingStartResult = void; diff --git a/packages/playwright-core/src/client/tracing.ts b/packages/playwright-core/src/client/tracing.ts index 4f403f5679a1c..928f8615e8133 100644 --- a/packages/playwright-core/src/client/tracing.ts +++ b/packages/playwright-core/src/client/tracing.ts @@ -42,7 +42,7 @@ export class Tracing extends ChannelOwner implements ap super(parent, type, guid, initializer); } - async start(options: { name?: string, title?: string, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, sources?: boolean, live?: boolean } = {}) { + async start(options: { name?: string, title?: string, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, coverage?: boolean, sources?: boolean, live?: boolean } = {}) { await this._wrapApiCall(async () => { this._includeSources = !!options.sources; this._isLive = !!options.live; @@ -53,6 +53,7 @@ export class Tracing extends ChannelOwner implements ap snapshotAria: snapshots.aria, snapshotScreen: snapshots.screen, screencast: options.screenshots, + coverage: options.coverage, live: options.live, }, kNoTimeout); const { traceName } = await this._channel.tracingStartChunk({ name: options.name, title: options.title }, kNoTimeout); diff --git a/packages/playwright-core/src/server/browserContext.ts b/packages/playwright-core/src/server/browserContext.ts index e510a16e645e9..f3e399f5b35c7 100644 --- a/packages/playwright-core/src/server/browserContext.ts +++ b/packages/playwright-core/src/server/browserContext.ts @@ -614,6 +614,10 @@ export abstract class BrowserContext extends Sdk this._origins.add(origin); } + visitedOrigins(): ReadonlySet { + return this._origins; + } + async storageState(progress: Progress, { indexedDB = false, opfs = false, credentials = false }: { indexedDB?: boolean, opfs?: boolean, credentials?: boolean } = {}): Promise { const result: channels.BrowserContextStorageStateResult = { cookies: await this.cookies(progress), @@ -648,25 +652,31 @@ export abstract class BrowserContext extends Sdk // If there are still origins to save, create a blank page to iterate over origins. if (originsToSave.size) { - const page = await this.newPage(progress, true /* forStorageState */); - try { - await page.addRequestInterceptor(progress, route => { - route.fulfill({ body: '' }).catch(() => {}); - }, 'prepend'); - for (const origin of originsToSave) { - const frame = page.mainFrame(); - await frame.gotoImpl(progress, origin, {}); - const storage: SerializedStorage = await frame.evaluateExpression(progress, collectScript, { world: 'utility' }); - if (hasStorage(storage)) - result.origins.push({ origin, ...storage }); - } - } finally { - await page.close(progress); - } + await this.visitOrigins(progress, originsToSave, async (frame, origin) => { + const storage: SerializedStorage = await frame.evaluateExpression(progress, collectScript, { world: 'utility' }); + if (hasStorage(storage)) + result.origins.push({ origin, ...storage }); + }); } return result; } + async visitOrigins(progress: Progress, origins: Set, callback: (frame: frames.Frame, origin: string) => Promise) { + const page = await this.newPage(progress, true /* forStorageState */); + try { + await page.addRequestInterceptor(progress, route => { + route.fulfill({ body: '' }).catch(() => {}); + }, 'prepend'); + for (const origin of origins) { + const frame = page.mainFrame(); + await frame.gotoImpl(progress, origin, {}); + await progress.race(callback(frame, origin)); + } + } finally { + await page.close(progress); + } + } + isCreatingStorageStatePage(): boolean { return this._creatingStorageStatePage; } diff --git a/packages/playwright-core/src/server/channels.d.ts b/packages/playwright-core/src/server/channels.d.ts index f2fc853c42368..7edbe1ba7fa5a 100644 --- a/packages/playwright-core/src/server/channels.d.ts +++ b/packages/playwright-core/src/server/channels.d.ts @@ -5076,6 +5076,7 @@ export type TracingTracingStartParams = { snapshotAria?: boolean, snapshotScreen?: boolean, screencast?: boolean, + coverage?: boolean, live?: boolean, }; export type TracingTracingStartOptions = { @@ -5084,6 +5085,7 @@ export type TracingTracingStartOptions = { snapshotAria?: boolean, snapshotScreen?: boolean, screencast?: boolean, + coverage?: boolean, live?: boolean, }; export type TracingTracingStartResult = void; diff --git a/packages/playwright-core/src/server/coverageRecorder.ts b/packages/playwright-core/src/server/coverageRecorder.ts new file mode 100644 index 0000000000000..2726546c6c46d --- /dev/null +++ b/packages/playwright-core/src/server/coverageRecorder.ts @@ -0,0 +1,147 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { kCoverageStashError, mergeIstanbulCoverage, sortedIstanbulCoverage } from '@isomorphic/istanbulCoverage'; +import * as rawCoverageSource from '../generated/coverageScriptSource'; + +import type { BrowserContext } from './browserContext'; +import type { InitScript, Page } from './page'; +import type { Progress } from './progress'; +import type { IstanbulCoverageChunk, IstanbulFileCoverage } from '@isomorphic/istanbulCoverage'; + +const kCoverageTakeName = '__pwCoverageTake'; + +const coverageTakeExpression = `window[${JSON.stringify(kCoverageTakeName)}] ? window[${JSON.stringify(kCoverageTakeName)}]() : []`; +const coverageDisposeExpression = `window[${JSON.stringify(kCoverageTakeName)}]?.dispose()`; + +export class CoverageRecorder { + private _context: BrowserContext; + private _coverage = new Map(); + private _stashedChunkIds = new Set(); + private _stashError: Error | undefined; + private _initScript: InitScript | undefined; + + constructor(context: BrowserContext) { + this._context = context; + } + + private _moduleExpression(call: string) { + return `(() => { + const module = {}; + ${rawCoverageSource.source} + ${call} + })()`; + } + + private _sessionId() { + return JSON.stringify(this._context.guid); + } + + async install(progress: Progress) { + if (this._initScript) + return; + const source = this._moduleExpression(`new (module.exports.CoverageScript())(window, ${JSON.stringify(kCoverageTakeName)}, ${this._sessionId()});`); + this._initScript = await this._context.addInitScript(progress, source); + await progress.race(this._context.safeNonStallingEvaluateInAllFrames(source, 'main')); + } + + async uninstall() { + await Promise.all([ + this._initScript?.dispose(), + this._context.safeNonStallingEvaluateInAllFrames(coverageDisposeExpression, 'main'), + ]).catch(() => {}); + } + + async flush(progress: Progress) { + await progress.race(Promise.all(this._context.pages().map(page => this.collectFromPage(page)))); + await this._harvestOriginsWithoutPage(progress); + } + + async take(progress: Progress, mode: 'keep' | 'discard'): Promise { + if (mode === 'discard') { + this._coverage.clear(); + return; + } + await this.flush(progress); + if (this._stashError) + throw this._stashError; + this._stashedChunkIds.clear(); + if (!this._coverage.size) + return; + const sorted = sortedIstanbulCoverage(this._coverage); + this._coverage.clear(); + return JSON.stringify(sorted); + } + + async collectFromPage(page: Page) { + if (page.isStorageStatePage) + return; + // A page that is going away may never answer. + await Promise.race([page.closedPromise, this._collectFromPage(page)]); + } + + private async _collectFromPage(page: Page) { + await Promise.all(page.frames().map(async frame => { + const chunks: string[] = await frame.nonStallingRawEvaluateInExistingMainContext(coverageTakeExpression).catch(error => this._takeFailed(error)); + for (const json of chunks) + this._append(json); + })); + } + + private async _harvestOriginsWithoutPage(progress: Progress) { + if (this._context.isClosingOrClosed()) + return; + const liveOrigins = new Set(); + for (const page of this._context.pages()) { + for (const frame of page.frames()) { + const origin = frame.origin(); + if (origin) + liveOrigins.add(origin); + } + } + // The origins that still have a page were drained by the sweep above. + const origins = new Set([...this._context.visitedOrigins()].filter(origin => !liveOrigins.has(origin))); + if (!origins.size) + return; + const source = this._moduleExpression(`return (module.exports.takeCoverageStashes())(window, ${this._sessionId()});`); + await this._context.visitOrigins(progress, origins, async frame => { + const chunks: string[] = await frame.evaluateExpression(progress, source, { world: 'main' }).catch(error => this._takeFailed(error)); + for (const json of chunks) + this._append(json); + }); + } + + // Evaluation fails for pages on their way out, only the stash error matters. + private _takeFailed(error: Error): string[] { + const message = error.message?.split('\n')[0].replace(/^Error: /, ''); + if (message?.includes(kCoverageStashError)) + this._stashError ??= new Error(message); + return []; + } + + private _append(json: string) { + try { + const chunk: IstanbulCoverageChunk = JSON.parse(json); + if (chunk.id) { + if (this._stashedChunkIds.has(chunk.id)) + return; + this._stashedChunkIds.add(chunk.id); + } + mergeIstanbulCoverage(this._coverage, chunk.data); + } catch { + } + } +} diff --git a/packages/playwright-core/src/server/debugController.ts b/packages/playwright-core/src/server/debugController.ts index 7ad8b2d4cc1da..48bca18fcce56 100644 --- a/packages/playwright-core/src/server/debugController.ts +++ b/packages/playwright-core/src/server/debugController.ts @@ -68,7 +68,7 @@ export class DebugController extends SdkObject { if (enabled && !this._trackHierarchyListener) { this._trackHierarchyListener = { onPageOpen: () => this._emitSnapshot(false), - onPageClose: () => this._emitSnapshot(false), + onPageDidClose: () => this._emitSnapshot(false), }; this._playwright.instrumentation.addListener(this._trackHierarchyListener, null); this._emitSnapshot(true); diff --git a/packages/playwright-core/src/server/instrumentation.ts b/packages/playwright-core/src/server/instrumentation.ts index a2de6a9806fbf..a9aa05b3f99ec 100644 --- a/packages/playwright-core/src/server/instrumentation.ts +++ b/packages/playwright-core/src/server/instrumentation.ts @@ -111,7 +111,8 @@ export interface Instrumentation { onCallLog(sdkObject: SdkObject, metadata: CallMetadata, logName: string, message: string): void; onAfterCall(progress: Progress, sdkObject: SdkObject): Promise; onPageOpen(page: Page): void; - onPageClose(page: Page): void; + onPageWillClose(page: Page): Promise; + onPageDidClose(page: Page): void; onBrowserOpen(browser: Browser): void; onBrowserClose(browser: Browser): void; onDialog(dialog: Dialog): void; @@ -124,7 +125,8 @@ export interface InstrumentationListener { onCallLog?(sdkObject: SdkObject, metadata: CallMetadata, logName: string, message: string): void; onAfterCall?(progress: Progress, sdkObject: SdkObject): Promise; onPageOpen?(page: Page): void; - onPageClose?(page: Page): void; + onPageWillClose?(page: Page): Promise; + onPageDidClose?(page: Page): void; onBrowserOpen?(browser: Browser): void; onBrowserClose?(browser: Browser): void; onDialog?(dialog: Dialog): void; diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.ts index 00ee7449d2fd0..0c5d10398148f 100644 --- a/packages/playwright-core/src/server/page.ts +++ b/packages/playwright-core/src/server/page.ts @@ -316,7 +316,7 @@ export class Page extends SdkObject { this.browserContext.emit(BrowserContext.Events.PageClosed, this); this.closedPromise.resolve(); if (!this.isStorageStatePage) - this.instrumentation.onPageClose(this); + this.instrumentation.onPageDidClose(this); } _didCrash() { @@ -831,6 +831,9 @@ export class Page extends SdkObject { if (options.reason) this._closeReason = options.reason; + if (!this.isStorageStatePage) + await this.instrumentation.onPageWillClose(this).catch(() => {}); + await this.screencast.handlePageOrContextClose(); if (this._lifecycle !== 'closing') { diff --git a/packages/playwright-core/src/server/playwright.ts b/packages/playwright-core/src/server/playwright.ts index dc129c7e23714..9b04c951a011b 100644 --- a/packages/playwright-core/src/server/playwright.ts +++ b/packages/playwright-core/src/server/playwright.ts @@ -56,7 +56,7 @@ export class Playwright extends SdkObject { onBrowserOpen: browser => this._allBrowsers.add(browser), onBrowserClose: browser => this._allBrowsers.delete(browser), onPageOpen: page => this._allPages.add(page), - onPageClose: page => this._allPages.delete(page), + onPageDidClose: page => this._allPages.delete(page), }, null); this.chromium = new Chromium(this, new BidiChromium(this)); this.firefox = new Firefox(this, new BidiFirefox(this)); diff --git a/packages/playwright-core/src/server/trace/recorder/tracing.ts b/packages/playwright-core/src/server/trace/recorder/tracing.ts index 3ba35b8348979..67b163b1fe5bf 100644 --- a/packages/playwright-core/src/server/trace/recorder/tracing.ts +++ b/packages/playwright-core/src/server/trace/recorder/tracing.ts @@ -30,6 +30,7 @@ import { getPlaywrightVersion } from '../../userAgent'; import { Snapshotter } from './snapshotter'; import { Artifact } from '../../artifact'; import { BrowserContext } from '../../browserContext'; +import { CoverageRecorder } from '../../coverageRecorder'; import { Dispatcher } from '../../dispatchers/dispatcher'; import { serializeError } from '../../errors'; import { HarRecorder } from '../../har/harRecorder'; @@ -63,6 +64,7 @@ export type TracerOptions = { snapshotAria?: boolean; snapshotScreen?: boolean; screencast?: boolean; + coverage?: boolean; live?: boolean; }; @@ -100,6 +102,7 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps private _contextCreatedEvent: trace.ContextCreatedTraceEvent; private _pendingHarEntries = new Set(); private _started = false; + private _coverageRecorder: CoverageRecorder | undefined; readonly harRecorders = new Map(); constructor(context: BrowserContext | APIRequestContext, tracesDir: string | undefined) { @@ -184,6 +187,8 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps // Tracing is 10x bigger if we include scripts in every trace. if (options.snapshotDom) this._harTracer.start({ omitScripts: !options.live }); + if (options.coverage && this._context instanceof BrowserContext) + this._coverageRecorder = new CoverageRecorder(this._context); this._started = true; } @@ -235,6 +240,8 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps this._harTracer.setOmitWebSocketFrames(!!process.env.PLAYWRIGHT_TRACING_NO_WEBSOCKET_FRAMES); if (this._state.options.snapshotDom) await this._snapshotter?.start(progress); + if (this._state.options.coverage) + await this._coverageRecorder?.install(progress); return { traceName: this._state.traceName }; } @@ -339,6 +346,9 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps this._closeAllGroups(); this._harTracer.stop(); this.flushHarEntries(); + const coverageRecorder = this._coverageRecorder; + this._coverageRecorder = undefined; + await coverageRecorder?.uninstall(); await this._fs.sync().finally(() => { this._state = undefined; }); @@ -387,12 +397,35 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps this._groupEnd(); } + // Collected before the chunk stops, while the pages can still be evaluated in. + private async _takeCoverage(progress: Progress, mode: TracingTracingStopChunkParams['mode']): Promise { + const recorder = this._coverageRecorder; + if (!this._state?.recording || !recorder) + return; + const coverageJson = await recorder.take(progress, mode === 'discard' ? 'discard' : 'keep'); + if (coverageJson === undefined) + return; + const coverageFile = path.join(this._state.tracesDir, `${this._state.traceName}-coverage-${this._state.chunkOrdinal}.json`); + this._fs.writeFile(coverageFile, coverageJson); + return coverageFile; + } + async stopChunk(progress: Progress, params: TracingTracingStopChunkParams): Promise<{ artifact?: Artifact, entries?: NameValue[] }> { if (this._isStopping) throw new Error(`Tracing is already stopping`); this._isStopping = true; try { - const result = this._stopChunk(params); + let coverageFile: string | undefined; + let coverageError: Error | undefined; + try { + coverageFile = await this._takeCoverage(progress, params.mode); + } catch (error) { + coverageError = error; + } + // The chunk is torn down either way, the coverage error is reported after. + const result = this._stopChunk(params, coverageFile); + if (coverageError) + throw coverageError; if (!result) return {}; @@ -421,7 +454,7 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps } } - private _stopChunk(params: TracingTracingStopChunkParams): { entries: NameValue[], zipFileName: string } | undefined { + private _stopChunk(params: TracingTracingStopChunkParams, coverageFile: string | undefined): { entries: NameValue[], zipFileName: string } | undefined { if (!this._state || !this._state.recording) { if (params.mode !== 'discard') throw new Error(`Must start tracing before stopping`); @@ -453,6 +486,8 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps const entries: NameValue[] = []; entries.push({ name: 'trace.trace', value: this._state.traceFile }); entries.push({ name: 'trace.network', value: newNetworkFile }); + if (coverageFile) + entries.push({ name: 'trace.coverage', value: coverageFile }); for (const file of new Set([...this._state.chunkFiles, ...this._state.crossChunkFiles])) entries.push({ name: file, value: path.join(this._state.tracesDir, file) }); @@ -488,6 +523,18 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps await this._captureScreenshot(progress, page, phase); if (options?.snapshotAria) await this._captureAriaSnapshot(progress, page, phase); + if (phase === 'after') + await this._captureCoverage(progress, page); + } + + private async _captureCoverage(progress: Progress, page: Page): Promise { + const recorder = this._coverageRecorder; + if (!recorder) + return; + try { + await progress.race(recorder.collectFromPage(page)); + } catch { + } } private _shouldCaptureAtPhase(metadata: CallMetadata, phase: trace.ActionPhase) { @@ -574,6 +621,10 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps return this._captureSnapshot(progress, sdkObject, 'after'); } + async onPageWillClose(page: Page) { + await this._coverageRecorder?.collectFromPage(page).catch(() => {}); + } + onEntryStarted(entry: har.Entry) { this._pendingHarEntries.add(entry); } @@ -670,7 +721,7 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps this._appendTraceEvent(event); } - onPageClose(page: Page) { + onPageDidClose(page: Page) { const event: trace.EventTraceEvent = { type: 'event', time: monotonicTime(), diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts index be0790ba1a826..5fcd6b1c96abe 100644 --- a/packages/playwright-core/types/types.d.ts +++ b/packages/playwright-core/types/types.d.ts @@ -23268,6 +23268,16 @@ export interface Tracing { * @param options */ start(options?: { + /** + * Whether to collect code coverage from istanbul-instrumented application code. Build the application with an + * istanbul instrumentation plugin, for example + * [`vite-plugin-istanbul`](https://www.npmjs.com/package/vite-plugin-istanbul) or + * [`babel-plugin-istanbul`](https://www.npmjs.com/package/babel-plugin-istanbul), so that pages expose the + * `window.__coverage__` object. Playwright collects accumulated counters from all pages and frames, including right + * before navigations and page closes, and stores them in istanbul format inside the trace file. + */ + coverage?: boolean; + /** * When enabled, the trace is written to an unarchived file that is updated in real time as actions occur, instead of * caching changes and archiving them into a zip file at the end. This is useful for live trace viewing during test diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index e139194f08e5e..a237ccd041251 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -291,7 +291,7 @@ export function toReporters(reporters: BuiltInReporter | ReporterDescription[] | return reporters; } -export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github', 'html', 'blob', 'perfetto'] as const; +export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github', 'html', 'blob', 'perfetto', 'coverage'] as const; export type BuiltInReporter = typeof builtInReporters[number]; export type ContextReuseMode = 'none' | 'when-possible'; diff --git a/packages/playwright/src/reporters/coverage.ts b/packages/playwright/src/reporters/coverage.ts new file mode 100644 index 0000000000000..f392f5b18d06f --- /dev/null +++ b/packages/playwright/src/reporters/coverage.ts @@ -0,0 +1,220 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs'; +import path from 'path'; + +import { mergeIstanbulCoverage, sortedIstanbulCoverage } from '@isomorphic/istanbulCoverage'; +import { ZipFile } from '@utils/zipFile'; +import { terminalScreen } from './base'; +import { resolveReporterOutputPath } from '../util'; + +import type { IstanbulCoverage, IstanbulFileCoverage } from '@isomorphic/istanbulCoverage'; +import type { CommonReporterOptions } from './base'; +import type { ReporterV2 } from './reporterV2'; +import type { TestCase, TestResult } from '../../types/testReporter'; + +type CoverageReporterOptions = { + outputDir?: string, +}; + +class CoverageReporter implements ReporterV2 { + private _options: CoverageReporterOptions & CommonReporterOptions; + private _traceFiles = new Set(); + + constructor(options: CoverageReporterOptions & CommonReporterOptions) { + this._options = options; + } + + version(): 'v2' { + return 'v2'; + } + + printsToStdio() { + return false; + } + + onTestEnd(test: TestCase, result: TestResult) { + for (const attachment of result.attachments) { + if (attachment.name === 'trace' && attachment.path) + this._traceFiles.add(attachment.path); + } + } + + async onEnd() { + const coverage = new Map(); + // Traces of several contexts are merged into one, each keeping its own coverage entry. + const traceFiles = [...this._traceFiles]; + while (traceFiles.length) { + const batch = traceFiles.splice(0, 16).map(async traceFile => { + const zipFile = new ZipFile(traceFile); + try { + const entries = (await zipFile.entries()).filter(entry => entry.match(/(^|-)trace\.coverage$/)); + return await Promise.all(entries.map(entry => zipFile.read(entry))); + } catch { + return []; + } finally { + zipFile.close(); + } + }); + for (const buffers of await Promise.all(batch)) { + for (const buffer of buffers) { + try { + mergeIstanbulCoverage(coverage, JSON.parse(buffer.toString('utf8'))); + } catch { + } + } + } + } + + if (!coverage.size) { + writeLine([ + `\nNo code coverage was collected.`, + `Make sure the application under test is built with istanbul instrumentation`, + `(e.g. vite-plugin-istanbul or babel-plugin-istanbul) so that pages expose window.__coverage__,`, + `and that tracing is configured with coverage, e.g. trace: { mode: 'on', coverage: true }.`, + `Coverage is aggregated from the tests whose traces are kept.`, + ].join('\n')); + return; + } + + const outputDir = resolveReporterOutputPath('coverage', this._options.configDir, this._options.outputDir); + await fs.promises.mkdir(outputDir, { recursive: true }); + const mergedJson = sortedIstanbulCoverage(coverage); + await fs.promises.writeFile(path.join(outputDir, 'coverage-final.json'), JSON.stringify(mergedJson)); + await fs.promises.writeFile(path.join(outputDir, 'lcov.info'), lcovReport(mergedJson)); + const hasHtml = this._tryWriteHtmlReport(mergedJson, outputDir); + + const summary = computeSummary(mergedJson); + const lines = [ + ``, + `Code coverage (${coverage.size} files):`, + ` statements: ${formatMetric(summary.statements)}`, + ` branches: ${formatMetric(summary.branches)}`, + ` functions: ${formatMetric(summary.functions)}`, + ` lines: ${formatMetric(summary.lines)}`, + `Coverage report written to ${path.relative(process.cwd(), outputDir)}`, + ]; + if (!hasHtml) + lines.push(`For an HTML report, install istanbul-lib-coverage, istanbul-lib-report and istanbul-reports, or run 'npx nyc report --temp-dir=${path.relative(process.cwd(), outputDir)} --reporter=html'.`); + writeLine(lines.join('\n')); + } + + // Istanbul report libraries are not shipped with Playwright, use the project's if any. + private _tryWriteHtmlReport(mergedJson: IstanbulCoverage, outputDir: string): boolean { + try { + const resolveFrom = (name: string) => require.resolve(name, { paths: [this._options.configDir] }); + const libCoverage = require(resolveFrom('istanbul-lib-coverage')); + const libReport = require(resolveFrom('istanbul-lib-report')); + const reports = require(resolveFrom('istanbul-reports')); + const context = libReport.createContext({ + dir: outputDir, + coverageMap: libCoverage.createCoverageMap(mergedJson), + }); + reports.create('html').execute(context); + return true; + } catch { + return false; + } + } +} + +type Metric = { covered: number, total: number }; + +function computeSummary(mergedJson: IstanbulCoverage) { + const statements: Metric = { covered: 0, total: 0 }; + const branches: Metric = { covered: 0, total: 0 }; + const functions: Metric = { covered: 0, total: 0 }; + const lines: Metric = { covered: 0, total: 0 }; + for (const fileCov of Object.values(mergedJson)) { + countHits(statements, Object.values(fileCov.s)); + countHits(functions, Object.values(fileCov.f)); + for (const counts of Object.values(fileCov.b)) + countHits(branches, counts); + countHits(lines, lineCoverage(fileCov).values()); + } + return { statements, branches, functions, lines }; +} + +function countHits(metric: Metric, counts: Iterable) { + for (const count of counts) { + ++metric.total; + if (count > 0) + ++metric.covered; + } +} + +function lineCoverage(fileCov: IstanbulFileCoverage): Map { + const lines = new Map(); + for (const [key, statement] of Object.entries(fileCov.statementMap)) { + const line = statement.start.line; + const count = fileCov.s[key] || 0; + lines.set(line, Math.max(lines.get(line) || 0, count)); + } + return lines; +} + +function formatMetric(metric: Metric) { + const percent = metric.total ? (metric.covered / metric.total * 100).toFixed(2) : '100.00'; + return `${percent}% (${metric.covered}/${metric.total})`; +} + +function lcovReport(mergedJson: IstanbulCoverage): string { + const out: string[] = []; + for (const [file, fileCov] of Object.entries(mergedJson)) { + out.push(`SF:${file}`); + let functionsCovered = 0; + for (const [key, fn] of Object.entries(fileCov.fnMap)) { + out.push(`FN:${fn.decl.start.line},${fn.name}`); + out.push(`FNDA:${fileCov.f[key] || 0},${fn.name}`); + if (fileCov.f[key] > 0) + ++functionsCovered; + } + out.push(`FNF:${Object.keys(fileCov.fnMap).length}`); + out.push(`FNH:${functionsCovered}`); + let branchesFound = 0; + let branchesCovered = 0; + for (const [key, branch] of Object.entries(fileCov.branchMap)) { + const counts: number[] = fileCov.b[key] || []; + branch.locations.forEach((location, i) => { + const count = counts[i] || 0; + out.push(`BRDA:${location.start.line ?? branch.loc.start.line},${key},${i},${count}`); + ++branchesFound; + if (count > 0) + ++branchesCovered; + }); + } + out.push(`BRF:${branchesFound}`); + out.push(`BRH:${branchesCovered}`); + const lines = lineCoverage(fileCov); + let linesCovered = 0; + for (const [line, count] of [...lines.entries()].sort((a, b) => a[0] - b[0])) { + out.push(`DA:${line},${count}`); + if (count > 0) + ++linesCovered; + } + out.push(`LF:${lines.size}`); + out.push(`LH:${linesCovered}`); + out.push(`end_of_record`); + } + return out.join('\n') + '\n'; +} + +function writeLine(line: string) { + terminalScreen.stdout.write(line + '\n'); +} + +export default CoverageReporter; diff --git a/packages/playwright/src/runner/reporters.ts b/packages/playwright/src/runner/reporters.ts index 01966f8b3d993..34cff70b12c59 100644 --- a/packages/playwright/src/runner/reporters.ts +++ b/packages/playwright/src/runner/reporters.ts @@ -19,6 +19,7 @@ import { calculateSha1 } from '@utils/crypto'; import { loadReporter } from './loadUtils'; import { formatError } from '../reporters/base'; import { BlobReporter } from '../reporters/blob'; +import CoverageReporter from '../reporters/coverage'; import DotReporter from '../reporters/dot'; import EmptyReporter from '../reporters/empty'; import GitHubReporter from '../reporters/github'; @@ -50,6 +51,7 @@ export async function createReporters(config: FullConfigInternal, mode: 'list' | 'junit': JUnitReporter, 'null': EmptyReporter, 'html': HtmlReporter, + 'coverage': CoverageReporter, }; const reporters: ReporterV2[] = []; descriptions ??= config.config.reporter; diff --git a/packages/playwright/src/worker/testTracing.ts b/packages/playwright/src/worker/testTracing.ts index fed713a572507..8b8637544eece 100644 --- a/packages/playwright/src/worker/testTracing.ts +++ b/packages/playwright/src/worker/testTracing.ts @@ -38,7 +38,7 @@ const version: trace.VERSION = 10; let traceOrdinal = 0; type TraceFixtureValue = PlaywrightWorkerOptions['trace'] | undefined; -type TraceOptions = { screenshots: boolean, snapshots: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, sources: boolean, attachments: boolean, live: boolean, mode: TraceMode }; +type TraceOptions = { screenshots: boolean, snapshots: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, coverage: boolean, sources: boolean, attachments: boolean, live: boolean, mode: TraceMode }; export class TestTracing { private _testInfo: TestInfoImpl; @@ -93,7 +93,7 @@ export class TestTracing { } async startIfNeeded(value: TraceFixtureValue) { - const defaultTraceOptions: TraceOptions = { screenshots: true, snapshots: true, sources: true, attachments: true, live: false, mode: 'off' }; + const defaultTraceOptions: TraceOptions = { screenshots: true, snapshots: true, coverage: false, sources: true, attachments: true, live: false, mode: 'off' }; if (!value) { this._options = defaultTraceOptions; diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 8d0eef73cc056..07effc78c1858 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -919,7 +919,7 @@ interface TestConfig { * ``` * */ - reporter?: LiteralUnion<'list'|'dot'|'line'|'github'|'json'|'junit'|'null'|'html'|'perfetto', string> | ReporterDescription[]; + reporter?: LiteralUnion<'list'|'dot'|'line'|'github'|'json'|'junit'|'null'|'html'|'perfetto'|'coverage', string> | ReporterDescription[]; /** * Global options for all tests, for example * [testOptions.browserName](https://playwright.dev/docs/api/class-testoptions#test-options-browser-name). Learn more @@ -7202,7 +7202,7 @@ export interface PlaywrightWorkerOptions { * * Learn more about [recording trace](https://playwright.dev/docs/test-use-options#recording-options). */ - trace: TraceMode | /** deprecated */ 'retry-with-trace' | { mode: TraceMode, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, sources?: boolean, attachments?: boolean }; + trace: TraceMode | /** deprecated */ 'retry-with-trace' | { mode: TraceMode, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, coverage?: boolean, sources?: boolean, attachments?: boolean }; /** * Whether to record video for each test. Defaults to `'off'`. The initial run of a test is the "first run"; * subsequent runs caused by [retries](https://playwright.dev/docs/test-retries) are "retries". diff --git a/packages/protocol/spec/tracing.yml b/packages/protocol/spec/tracing.yml index de9a644d9d287..054e06d8bb627 100644 --- a/packages/protocol/spec/tracing.yml +++ b/packages/protocol/spec/tracing.yml @@ -26,6 +26,7 @@ Tracing: snapshotAria: boolean? snapshotScreen: boolean? screencast: boolean? + coverage: boolean? live: boolean? tracingStartChunk: diff --git a/packages/protocol/src/validator.ts b/packages/protocol/src/validator.ts index 21e7cbe6036e3..86f1e91255607 100644 --- a/packages/protocol/src/validator.ts +++ b/packages/protocol/src/validator.ts @@ -3065,6 +3065,7 @@ scheme.TracingTracingStartParams = tObject({ snapshotAria: tOptional(tBoolean), snapshotScreen: tOptional(tBoolean), screencast: tOptional(tBoolean), + coverage: tOptional(tBoolean), live: tOptional(tBoolean), }); scheme.TracingTracingStartResult = tOptional(tObject({})); diff --git a/tests/config/utils.ts b/tests/config/utils.ts index b28fc36365428..0a4148ba7747a 100644 --- a/tests/config/utils.ts +++ b/tests/config/utils.ts @@ -104,6 +104,18 @@ export function suppressCertificateWarning() { }; } +// Traces of several contexts are merged into one, each keeping its own coverage entry. +export async function parseTraceCoverage(file: string): Promise { + const { resources } = await parseTraceRaw(file); + const entries = [...resources.keys()].filter(name => name.match(/(^|-)trace\.coverage$/)); + if (!entries.length) + return undefined; + const coverage = new Map(); + for (const entry of entries) + iso.mergeIstanbulCoverage(coverage, JSON.parse(resources.get(entry)!.toString())); + return iso.sortedIstanbulCoverage(coverage); +} + export async function parseTraceRaw(file: string): Promise<{ events: any[], resources: Map, actions: string[], actionObjects: ActionTraceEvent[], stacks: Map }> { const zipFS = new utils.ZipFile(file); const resources = new Map(); diff --git a/tests/library/coverage.spec.ts b/tests/library/coverage.spec.ts new file mode 100644 index 0000000000000..53441d41b1537 --- /dev/null +++ b/tests/library/coverage.spec.ts @@ -0,0 +1,312 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { browserTest as it, expect } from '../config/browserTest'; +import { parseTraceCoverage } from '../config/utils'; + +const fileCoverage = (file: string, s0: number) => ({ + [file]: { + path: file, + statementMap: { '0': { start: { line: 1, column: 0 }, end: { line: 1, column: 20 } } }, + fnMap: {}, + branchMap: {}, + s: { '0': s0 }, + f: {}, + b: {}, + }, +}); + +const coverageScript = (file: string, s0: number) => ``; + +it('should collect istanbul coverage into the trace', async ({ browser }, testInfo) => { + const context = await browser.newContext(); + await context.tracing.start({ coverage: true }); + + const page = await context.newPage(); + await page.setContent(coverageScript('a.js', 3)); + + // Coverage of a closed page is collected automatically. + const page2 = await context.newPage(); + await page2.setContent(coverageScript('b.js', 2)); + await page2.close(); + + const traceFile = testInfo.outputPath('trace.zip'); + await context.tracing.stop({ path: traceFile }); + await context.close(); + + const data = await parseTraceCoverage(traceFile); + expect(data['a.js'].s['0']).toBe(3); + expect(data['b.js'].s['0']).toBe(2); +}); + +it('should collect coverage per trace chunk', async ({ browser }, testInfo) => { + const context = await browser.newContext(); + await context.tracing.start({ coverage: true }); + const page = await context.newPage(); + + await page.setContent(coverageScript('a.js', 5)); + const traceFile1 = testInfo.outputPath('trace1.zip'); + await context.tracing.stopChunk({ path: traceFile1 }); + + await context.tracing.startChunk(); + await page.setContent(coverageScript('b.js', 7)); + const traceFile2 = testInfo.outputPath('trace2.zip'); + await context.tracing.stopChunk({ path: traceFile2 }); + + await context.tracing.stop(); + await context.close(); + + const data1 = await parseTraceCoverage(traceFile1); + expect(data1['a.js'].s['0']).toBe(5); + expect(data1['b.js']).toBe(undefined); + + const data2 = await parseTraceCoverage(traceFile2); + expect(data2['b.js'].s['0']).toBe(7); + expect(data2['a.js']).toBe(undefined); +}); + +it('should report maps once and counters incrementally', async ({ browser, server }) => { + const context = await browser.newContext(); + await context.tracing.start({ coverage: true }); + const page = await context.newPage(); + await page.goto(server.EMPTY_PAGE); + + // Actions collect the counters on their own, so hit and collect in one evaluate. + const first = await page.evaluate(coverage => { + (window as any).__coverage__ = JSON.parse(coverage); + return (window as any).__pwCoverageTake().map((json: string) => JSON.parse(json)); + }, JSON.stringify(fileCoverage('a.js', 3))); + expect(first[0].data['a.js'].statementMap).toBeTruthy(); + expect(first[0].data['a.js'].s).toEqual({ '0': 3 }); + + const second = await page.evaluate(() => { + (window as any).__coverage__['a.js'].s['0'] += 2; + return (window as any).__pwCoverageTake().map((json: string) => JSON.parse(json)); + }); + expect(second[0].data['a.js'].statementMap).toBe(undefined); + expect(second[0].data['a.js'].s).toEqual({ '0': 2 }); + + // Nothing was hit since the last report. + expect(await page.evaluate(() => (window as any).__pwCoverageTake())).toEqual([]); + + await context.tracing.stop(); + await context.close(); +}); + +it('should accumulate counters across pulls and keep never hit files', async ({ browser }, testInfo) => { + const context = await browser.newContext(); + await context.tracing.start({ coverage: true }); + const page = await context.newPage(); + await page.setContent(coverageScript('a.js', 0)); + await page.evaluate(() => (window as any).__coverage__['a.js'].s['0'] += 4); + await page.evaluate(() => (window as any).__coverage__['a.js'].s['0'] += 3); + + const traceFile = testInfo.outputPath('trace.zip'); + await context.tracing.stop({ path: traceFile }); + await context.close(); + + const data = await parseTraceCoverage(traceFile); + expect(data['a.js'].s['0']).toBe(7); + expect(Object.keys(data['a.js'].statementMap)).toEqual(['0']); +}); + +async function openPopup(page: any, server: any, file: string, s0: number) { + const [popup] = await Promise.all([ + page.waitForEvent('popup'), + page.evaluate(url => (window as any).__popup = window.open(url), server.EMPTY_PAGE), + ]); + await popup.evaluate(coverage => (window as any).__coverage__ = JSON.parse(coverage), JSON.stringify(fileCoverage(file, s0))); + return popup; +} + +it('should collect coverage of a page closed by in-page script', async ({ browser, server }, testInfo) => { + const context = await browser.newContext(); + await context.tracing.start({ coverage: true }); + const page = await context.newPage(); + await page.goto(server.EMPTY_PAGE); + + const popup = await openPopup(page, server, 'popup.js', 5); + // An in-page close never reaches Playwright as page.close(), the counters + // are preserved because the actions collect them as they go. + await Promise.all([ + popup.waitForEvent('close'), + popup.evaluate(() => setTimeout(() => window.close(), 0)), + ]); + + const traceFile = testInfo.outputPath('trace.zip'); + await context.tracing.stop({ path: traceFile }); + await context.close(); + + const data = await parseTraceCoverage(traceFile); + expect(data['popup.js'].s['0']).toBe(5); +}); + +it('should not double count a stash picked up twice', async ({ browser, server }, testInfo) => { + const context = await browser.newContext(); + await context.tracing.start({ coverage: true }); + const page = await context.newPage(); + await page.goto(server.EMPTY_PAGE); + + // Emulate two documents picking up the same stash before either removes it. + const prefix = '__pwCoverage.' + (context as any)._guid + '.'; + await page.evaluate(({ prefix, coverage }) => { + const chunk = JSON.stringify({ id: 'stash-id', data: JSON.parse(coverage) }); + localStorage.setItem(prefix + 'one', chunk); + localStorage.setItem(prefix + 'two', chunk); + }, { prefix, coverage: JSON.stringify(fileCoverage('stashed.js', 4)) }); + + const traceFile = testInfo.outputPath('trace.zip'); + await context.tracing.stop({ path: traceFile }); + await context.close(); + + const data = await parseTraceCoverage(traceFile); + expect(data['stashed.js'].s['0']).toBe(4); +}); + +it('should discard stashes of other sessions', async ({ browser, server }, testInfo) => { + const context = await browser.newContext(); + await context.tracing.start({ coverage: true }); + const page = await context.newPage(); + await page.goto(server.EMPTY_PAGE); + await page.evaluate(coverage => { + localStorage.setItem('__pwCoverage.other-session.1', JSON.stringify({ id: 'other', data: JSON.parse(coverage) })); + }, JSON.stringify(fileCoverage('stale.js', 7))); + + const remaining = await page.evaluate(() => Object.keys(localStorage).filter(key => key.startsWith('__pwCoverage.'))); + expect(remaining).toEqual([]); + + const traceFile = testInfo.outputPath('trace.zip'); + await context.tracing.stop({ path: traceFile }); + await context.close(); + + // The stale stash was the only coverage, so nothing was collected at all. + expect(await parseTraceCoverage(traceFile)).toBe(undefined); +}); + +it('should collect coverage of an origin left without a page', async ({ browser, server }, testInfo) => { + const context = await browser.newContext(); + await context.tracing.start({ coverage: true }); + const page = await context.newPage(); + await page.goto(server.EMPTY_PAGE); + + const popup = await openPopup(page, server, 'popup.js', 0); + await popup.evaluate(url => { + (window as any).__go = () => { + (window as any).__coverage__['popup.js'].s['0'] = 5; + window.location.href = url; + }; + }, server.CROSS_PROCESS_PREFIX + '/empty.html'); + // The popup is hit and navigates away, leaving a stash behind. Triggered from + // the opener, so that the pull after the action does not race the navigation. + await Promise.all([ + popup.waitForURL(server.CROSS_PROCESS_PREFIX + '/empty.html'), + page.evaluate(() => (window as any).__popup.__go()), + ]); + // Leave the origin of the stash without a page to relay it. + await page.goto(server.CROSS_PROCESS_PREFIX + '/empty.html'); + + const traceFile = testInfo.outputPath('trace.zip'); + await context.tracing.stop({ path: traceFile }); + await context.close(); + + const data = await parseTraceCoverage(traceFile); + expect(data['popup.js'].s['0']).toBe(5); +}); + +it('should pull counters as the actions go', async ({ browser, server }, testInfo) => { + const context = await browser.newContext(); + await context.tracing.start({ coverage: true }); + const page = await context.newPage(); + await page.goto(server.EMPTY_PAGE); + await page.setContent(coverageScript('a.js', 3)); + // The counters are collected by the action itself, not only by the final flush. + expect(await page.evaluate(() => (window as any).__coverage__['a.js'].s['0'])).toBe(0); + + const traceFile = testInfo.outputPath('trace.zip'); + await context.tracing.stop({ path: traceFile }); + await context.close(); + + const data = await parseTraceCoverage(traceFile); + expect(data['a.js'].s['0']).toBe(3); +}); + +it('should not collect coverage without the option', async ({ browser }, testInfo) => { + const context = await browser.newContext(); + await context.tracing.start(); + const page = await context.newPage(); + await page.setContent(coverageScript('a.js', 1)); + + const traceFile = testInfo.outputPath('trace.zip'); + await context.tracing.stop({ path: traceFile }); + await context.close(); + + expect(await parseTraceCoverage(traceFile)).toBe(undefined); +}); + +it('should count only the hits after start', async ({ browser, server }, testInfo) => { + const context = await browser.newContext(); + const page = await context.newPage(); + await page.goto(server.EMPTY_PAGE); + await page.evaluate(coverage => (window as any).__coverage__ = JSON.parse(coverage), JSON.stringify(fileCoverage('a.js', 3))); + + await context.tracing.start({ coverage: true }); + await page.evaluate(() => (window as any).__coverage__['a.js'].s['0'] += 2); + const traceFile = testInfo.outputPath('trace.zip'); + await context.tracing.stop({ path: traceFile }); + await context.close(); + + const data = await parseTraceCoverage(traceFile); + expect(data['a.js'].s['0']).toBe(2); + expect(Object.keys(data['a.js'].statementMap)).toEqual(['0']); +}); + +it('should stop collecting when tracing stops', async ({ browser, server }, testInfo) => { + const context = await browser.newContext(); + await context.tracing.start({ coverage: true }); + const page = await context.newPage(); + await page.goto(server.EMPTY_PAGE); + await page.evaluate(coverage => (window as any).__coverage__ = JSON.parse(coverage), JSON.stringify(fileCoverage('a.js', 1))); + await context.tracing.stop(); + + expect(await page.evaluate(() => typeof (window as any).__pwCoverageTake)).toBe('undefined'); + // Leaving the document no longer stashes the counters. + await page.evaluate(() => (window as any).__coverage__['a.js'].s['0'] = 5); + await page.goto(server.PREFIX + '/title.html'); + expect(await page.evaluate(() => Object.keys(localStorage).filter(key => key.startsWith('__pwCoverage.')))).toEqual([]); + expect(await page.evaluate(() => typeof (window as any).__pwCoverageTake)).toBe('undefined'); + await context.close(); +}); + +it('should surface a failure to stash the coverage', async ({ browser, server }, testInfo) => { + const context = await browser.newContext(); + await context.tracing.start({ coverage: true }); + const page = await context.newPage(); + await page.goto(server.EMPTY_PAGE); + await page.evaluate(coverage => { + (window as any).__coverage__ = JSON.parse(coverage); + // The stash does not fit, the error record still does. + const setItem = Storage.prototype.setItem; + Storage.prototype.setItem = function(key: string, value: string) { + if (value.length > 100) + throw new DOMException('The quota has been exceeded.', 'QuotaExceededError'); + setItem.call(this, key, value); + }; + window.dispatchEvent(new Event('pagehide')); + }, JSON.stringify(fileCoverage('a.js', 3))); + + await expect(context.tracing.stop({ path: testInfo.outputPath('trace.zip') })).rejects.toThrow('Failed to stash the coverage: QuotaExceededError: The quota has been exceeded.'); + await context.close(); +}); diff --git a/tests/playwright-test/coverage.spec.ts b/tests/playwright-test/coverage.spec.ts new file mode 100644 index 0000000000000..f00a17b582ef1 --- /dev/null +++ b/tests/playwright-test/coverage.spec.ts @@ -0,0 +1,150 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs'; +import { test, expect } from './playwright-test-fixtures'; +import { parseTraceCoverage } from '../config/utils'; + +const coverageHelper = ` + const makeCoverage = (file, s0) => ({ + [file]: { + path: file, + statementMap: { + '0': { start: { line: 1, column: 0 }, end: { line: 1, column: 20 } }, + '1': { start: { line: 2, column: 0 }, end: { line: 2, column: 20 } }, + }, + fnMap: { + '0': { name: 'foo', decl: { start: { line: 1, column: 0 }, end: { line: 1, column: 3 } }, loc: { start: { line: 1, column: 0 }, end: { line: 3, column: 1 } }, line: 1 }, + }, + branchMap: { + '0': { loc: { start: { line: 2, column: 0 }, end: { line: 2, column: 20 } }, type: 'if', locations: [{ start: { line: 2, column: 0 }, end: { line: 2, column: 10 } }, { start: { line: 2, column: 10 }, end: { line: 2, column: 20 } }], line: 2 }, + }, + s: { '0': s0, '1': 0 }, + f: { '0': 1 }, + b: { '0': [1, 0] }, + }, + }); + const coverageScript = (file, s0) => ''; +`; + +test('should collect istanbul coverage into the trace', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + use: { trace: { mode: 'on', coverage: true } }, + }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + ${coverageHelper} + test('pass', async ({ page }) => { + const http = require('http'); + const server = http.createServer((req, res) => { + res.setHeader('Content-Type', 'text/html'); + if (req.url === '/frame') + res.end(coverageScript('src/frame.js', 3)); + else if (req.url === '/page1') + res.end(coverageScript('src/app.js', 5) + ''); + else + res.end(coverageScript('src/app.js', 2)); + }); + await new Promise(f => server.listen(0, '127.0.0.1', f)); + const origin = 'http://127.0.0.1:' + server.address().port; + await page.goto(origin + '/page1'); + await expect(page.locator('iframe')).toBeVisible(); + // Navigating away parks the first document's counters for the next one. + await page.goto(origin + '/page2'); + server.close(); + }); + `, + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + + const data = await parseTraceCoverage(test.info().outputPath('test-results', 'a-pass', 'trace.zip')); + expect(data['src/app.js'].s['0']).toBe(7); + expect(data['src/app.js'].s['1']).toBe(0); + expect(data['src/app.js'].f['0']).toBe(2); + expect(data['src/app.js'].b['0']).toEqual([2, 0]); + expect(data['src/frame.js'].s['0']).toBe(3); +}); + +test('should aggregate coverage with the coverage reporter', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + use: { trace: { mode: 'on', coverage: true } }, + reporter: [['dot'], ['coverage', { outputDir: 'coverage-report' }]], + }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + ${coverageHelper} + test('first', async ({ page }) => { + await page.setContent(coverageScript('src/app.js', 5)); + }); + `, + 'b.test.ts': ` + import { test, expect } from '@playwright/test'; + ${coverageHelper} + test('second', async ({ page, browser }) => { + await page.setContent(coverageScript('src/app.js', 2)); + // Every context of a test keeps its own coverage entry in the trace. + const second = await browser.newContext(); + await (await second.newPage()).setContent(coverageScript('src/second.js', 4)); + await second.close(); + }); + `, + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(2); + + const final = JSON.parse(fs.readFileSync(test.info().outputPath('coverage-report', 'coverage-final.json'), 'utf8')); + expect(final['src/app.js'].s['0']).toBe(7); + expect(final['src/app.js'].s['1']).toBe(0); + expect(final['src/app.js'].f['0']).toBe(2); + expect(final['src/app.js'].b['0']).toEqual([2, 0]); + expect(final['src/second.js'].s['0']).toBe(4); + + const lcov = fs.readFileSync(test.info().outputPath('coverage-report', 'lcov.info'), 'utf8'); + expect(lcov).toContain('SF:src/app.js'); + expect(lcov).toContain('DA:1,7'); + expect(lcov).toContain('FNDA:2,foo'); + expect(lcov).toContain('SF:src/second.js'); + + expect(result.output).toContain('Code coverage (2 files)'); + expect(result.output).toContain('statements: 50.00% (2/4)'); +}); + +test('should warn when no coverage was collected', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + use: { trace: { mode: 'on', coverage: true } }, + reporter: [['dot'], ['coverage', { outputDir: 'coverage-report' }]], + }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('no instrumented code', async ({ page }) => { + await page.setContent('
hello
'); + }); + `, + }); + expect(result.exitCode).toBe(0); + expect(result.output).toContain('No code coverage was collected'); + expect(fs.existsSync(test.info().outputPath('coverage-report', 'coverage-final.json'))).toBe(false); +}); diff --git a/utils/generate_injected.js b/utils/generate_injected.js index 4162e312d2e03..86fb7cb39d3a1 100644 --- a/utils/generate_injected.js +++ b/utils/generate_injected.js @@ -81,6 +81,11 @@ const injectedScripts = [ path.join(ROOT, 'packages', 'injected', 'lib'), path.join(ROOT, 'packages', 'playwright-core', 'src', 'generated'), ], + [ + path.join(ROOT, 'packages', 'injected', 'src', 'coverageScript.ts'), + path.join(ROOT, 'packages', 'injected', 'lib'), + path.join(ROOT, 'packages', 'playwright-core', 'src', 'generated'), + ], ]; const modulePrefix = ` diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 90a983fdcea29..4a829c80ff689 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -69,7 +69,7 @@ type LiteralUnion = T | (U & { zz_IGNORE_ME?: never }); interface TestConfig { projects?: Project[]; - reporter?: LiteralUnion<'list'|'dot'|'line'|'github'|'json'|'junit'|'null'|'html'|'perfetto', string> | ReporterDescription[]; + reporter?: LiteralUnion<'list'|'dot'|'line'|'github'|'json'|'junit'|'null'|'html'|'perfetto'|'coverage', string> | ReporterDescription[]; use?: UseOptions; webServer?: TestConfigWebServer | TestConfigWebServer[]; } @@ -272,7 +272,7 @@ export interface PlaywrightWorkerOptions { connectOptions: ConnectOptions | undefined; reuseContext: boolean; screenshot: ScreenshotMode | { mode: ScreenshotMode } & Pick; - trace: TraceMode | /** deprecated */ 'retry-with-trace' | { mode: TraceMode, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, sources?: boolean, attachments?: boolean }; + trace: TraceMode | /** deprecated */ 'retry-with-trace' | { mode: TraceMode, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, coverage?: boolean, sources?: boolean, attachments?: boolean }; video: VideoMode | /** deprecated */ 'retry-with-video' | { mode: VideoMode, size?: ViewportSize, show?: { actions?: { duration?: number, position?: 'top-left' | 'top' | 'top-right' | 'bottom-left' | 'bottom' | 'bottom-right', fontSize?: number, cursor?: 'none' | 'pointer' }, test?: { level?: 'file' | 'title' | 'step', position?: 'top-left' | 'top' | 'top-right' | 'bottom-left' | 'bottom' | 'bottom-right', fontSize?: number } } }; }