diff --git a/.gitattributes b/.gitattributes index 59dd302286..9a057c9df0 100644 --- a/.gitattributes +++ b/.gitattributes @@ -32,3 +32,5 @@ Darling/Darling.Tests/Fixtures/ManagedConf/rehearsal-postgresql.conf text eol=lf tools/pg-verification-rig/Dockerfile text eol=lf tools/pg-verification-rig/*.yml text eol=lf tools/pg-verification-rig/*.sql text eol=lf +# Screenshot baselines for the web UI check (#5744): PNGs, never diffed or converted as text. +tools/ui-smoke/baselines/*.png binary diff --git a/.github/workflows/darling-ui-smoke.yml b/.github/workflows/darling-ui-smoke.yml index ef3ef20113..d0cfb273a5 100644 --- a/.github/workflows/darling-ui-smoke.yml +++ b/.github/workflows/darling-ui-smoke.yml @@ -2,7 +2,8 @@ name: Darling UI smoke # #5701 phase 1: loads the Darling web pages and Web Admin in a real Chrome against a service on a throwaway, # seeded PostgreSQL store, and fails on a console error, a failed request, an error strip, or a panel or grid that -# the seed says must have data and does not. No screenshots are compared. See tools/ui-smoke/ (run-ui-smoke.ps1 is +# the seed says must have data and does not. #5744 (phase 2): four pages are also compared with screenshot baselines +# (tools/ui-smoke/baselines/*.png, in Playwright's own pinned Chromium). See tools/ui-smoke/ (run-ui-smoke.ps1 is # the entry, locally and here). # # Triggers: a pull request touching the web files, the harness or this workflow (a draft pull request is skipped @@ -20,7 +21,15 @@ on: - '.github/workflows/darling-ui-smoke.yml' # The nightly runs it as a reusable workflow (nightly.yml, outside publish's needs). workflow_call: + # update_baselines: render the four screenshot pages and upload new PNGs as the artifact "darling-ui-smoke-new-baselines" + # (nothing is compared and nothing is committed). Baselines come from this CI image only: download the artifact and commit + # the PNGs to tools/ui-smoke/baselines/ on the pull request, where GitHub's image diff is the review. workflow_dispatch: + inputs: + update_baselines: + description: 'Write new screenshot baselines (artifact darling-ui-smoke-new-baselines) instead of comparing' + type: boolean + default: false permissions: contents: read @@ -35,8 +44,13 @@ jobs: # Skip a draft pull request; a dispatch has no pull request and always runs. if: github.event_name != 'pull_request' || github.event.pull_request.draft == false runs-on: windows-latest - # Expected 8 to 10 minutes warm (service build 2, PostgreSQL 1, browser walk 2 to 3); 3 more on a cold pg-runtime cache. + # Measured (#5744, five green runs with both caches warm): 3.4 to 5.8 minutes in all. A typical run: checkout and .NET 0.5, service build + # 1 to 1.2, PostgreSQL 0.2, browser walk and the stale wait 1.7, four screenshots 0.1. A cold pg-runtime cache adds 3. The 20 stays: over 3 times the slowest run. timeout-minutes: 20 + # One explicit home for Playwright's Chromium. The `env` context holds only variables declared in an env: map (never the runner's + # own LOCALAPPDATA), so the cache step and the install in run-ui-smoke.ps1 both use this path, not Playwright's default. + env: + PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}\.playwright-browsers steps: - uses: actions/checkout@v7 @@ -101,10 +115,22 @@ jobs: & "$bin\createdb.exe" -h 127.0.0.1 -p 5541 -U darling darling if ($LASTEXITCODE -ne 0) { throw "createdb failed ($LASTEXITCODE)" } + # Playwright's own Chromium for the screenshot pages, pinned by the playwright-core version in package-lock.json and cached by it, + # at PLAYWRIGHT_BROWSERS_PATH (job env above), which `playwright-core install chromium` and chromium.launch() both honour. + - name: Cache Playwright Chromium + uses: actions/cache@v6 + with: + path: ${{ env.PLAYWRIGHT_BROWSERS_PATH }} + key: playwright-chromium-${{ runner.os }}-${{ hashFiles('tools/ui-smoke/package-lock.json') }} + - name: Run the web UI smoke check shell: pwsh + env: + UPDATE_BASELINES: ${{ github.event_name == 'workflow_dispatch' && inputs.update_baselines }} run: | - ./tools/ui-smoke/run-ui-smoke.ps1 -PgBin "$env:GITHUB_WORKSPACE\Darling\artifacts\pg-runtime\pgsql\bin" -PgPort 5541 -WorkDir "$env:RUNNER_TEMP\ui-smoke" -NoBuild -Channel chrome + $update = @{ } + if ($env:UPDATE_BASELINES -eq 'true') { $update.UpdateBaselines = $true } + ./tools/ui-smoke/run-ui-smoke.ps1 -PgBin "$env:GITHUB_WORKSPACE\Darling\artifacts\pg-runtime\pgsql\bin" -PgPort 5541 -WorkDir "$env:RUNNER_TEMP\ui-smoke" -NoBuild -Channel chrome @update - name: Logs tail if: always() @@ -128,6 +154,26 @@ jobs: retention-days: 7 if-no-files-found: ignore + # An update_baselines dispatch: the new PNGs, one per screenshot page, to download and commit. + - name: Upload new screenshot baselines + if: always() && github.event_name == 'workflow_dispatch' && inputs.update_baselines + uses: actions/upload-artifact@v6 + with: + name: darling-ui-smoke-new-baselines + path: ${{ runner.temp }}\ui-smoke\out\new-baselines\*.png + retention-days: 14 + if-no-files-found: warn + + # A compare that failed: for each page that differs, the baseline, the new screenshot and the difference image. + - name: Upload screenshot differences + if: failure() + uses: actions/upload-artifact@v6 + with: + name: darling-ui-smoke-screenshot-diffs + path: ${{ runner.temp }}\ui-smoke\out\shot-failures\*.png + retention-days: 14 + if-no-files-found: ignore + - name: Stop throwaway PostgreSQL if: always() shell: pwsh diff --git a/tools/ui-smoke/README.md b/tools/ui-smoke/README.md index 796a4730c9..2bbab09ea1 100644 --- a/tools/ui-smoke/README.md +++ b/tools/ui-smoke/README.md @@ -2,8 +2,8 @@ Loads every Darling web page and Web Admin tab in a real Chrome against a service running on a throwaway, seeded PostgreSQL store. It fails the job on a console error, a page error, a failed request, an answer of 400 or more, an -error strip, or a panel or grid that the seed says must have data and does not. No screenshots are compared: a screenshot -and a trace are kept only for a page that failed. +error strip, or a panel or grid that the seed says must have data and does not. A screenshot and a trace are kept for a +page that failed. Four pages are also compared with screenshot baselines (see "Screenshot baselines" below). It runs in `.github/workflows/darling-ui-smoke.yml` ("Darling web UI smoke") on a pull request that touches `Darling/PerformanceMonitor.Darling.Service/wwwroot/**`, `tools/ui-smoke/**` or the workflow (draft pull requests are @@ -19,6 +19,8 @@ reports and does not stop publishing). It is not a required check: a path-filter | `seed.sql` | The rows. Every time is relative to the store's own clock and every count is fixed, so the assertions are exact. | | `pages.json` | The pages and what each must show. | | `smoke.mjs` | The browser walk (`playwright-core`, the installed Chrome or Edge). | +| `shots.mjs` | The screenshot step (`playwright-core`'s own Chromium, `pixelmatch`, `pngjs`). | +| `baselines/*.png` | The baseline screenshots, one per page marked `screenshot` in `pages.json`. | ## What it checks, per page @@ -52,6 +54,46 @@ Logs, `report.json`, failure screenshots and traces land in `-WorkDir` (default trace with `npx playwright show-trace .trace.zip`. `UI_SMOKE_PAGES=ui-pg-waits,alerts` runs only those pages when you call `node smoke.mjs` yourself. +## Screenshot baselines + +Four pages have a baseline PNG in `baselines/`: `fleet`, `ui-sql-overview`, `ui-sql-queries` and `admin-servers`. After the +smoke walk, `shots.mjs` renders each one at 1280 by 800 and compares it with its baseline. The check fails when any pixel +differs by more than a small color threshold. The renders are the same from run to run, so there is no allowance. A page is marked by a `screenshot` key in `pages.json`. +The key lists CSS selectors to paint over, for text that changes from run to run. + +How the page is kept the same on every run: + +- The browser is Playwright's own Chromium, pinned by the `playwright-core` version. It is not the runner's Chrome, which + updates with the runner image. +- The viewport, scale (1), color scheme, locale (en-US), time zone (UTC) and reduced motion are fixed. Animations are off and + the caret is hidden. +- The seed dates every row from one anchor, the minute `run-ui-smoke.ps1` ran it. The service must keep its real clock, so the + browser is moved instead. Every `/api` answer has its timestamps shifted to a fixed date. Every windowed read gets an + `as_of` of anchor plus one minute. The page clock is pinned to the same fixed date plus one minute. +- A screenshot is taken again until two in a row are identical. If that does not happen in 8 tries, the page fails. +- The service marks the two unreachable servers "collection stale" a couple of minutes after the seed anchor, and "offline" + after 30. The shots wait until `/api/fleet` shows every card stale. They give up after 4 minutes with one message if the + cards never got there, and with another if the cards are already past stale. +- The database size in the status bar is not the same on every run. Its width changes when it gains a digit (99 MB to 100 MB). + In one run the item was missing, because the read of the size had not finished. A mask hides the text, not the box. For the + shots, `shots.mjs` adds a style that hides the item and the separator before it, so the bar is the same either way. +- `report.json` is `{ pages, screenshotProblems }`. The page walk is written first, then rewritten with the screenshot + problems, so a failure in the screenshot step cannot lose the walk. +- Playwright's Chromium lives in the folder named by `PLAYWRIGHT_BROWSERS_PATH`. The workflow sets it and caches that folder. + +To refresh the baselines (after a change to the page that you want to keep): + +1. Start the workflow "Darling UI smoke" by hand on your branch, with `update_baselines` set. For example: + `gh workflow run darling-ui-smoke.yml --ref -f update_baselines=true`. +2. Download the artifact `darling-ui-smoke-new-baselines` from that run. +3. Copy the PNGs into `tools/ui-smoke/baselines/` and commit them on the pull request. +4. Review GitHub's image diff of each PNG. Never commit a PNG that you rendered on your own machine. Fonts and the browser + differ, so most pixels differ in CI. + +When a compare fails, the job uploads the artifact `darling-ui-smoke-screenshot-diffs`. For each page that differs, it holds +`.baseline.png`, `.actual.png` and `.diff.png` (the changed pixels). If the change is wanted, refresh the +baselines as above. If a runner image update moved the pixels, refresh them the same way. + ## Add a page or a seeded table - A page: one entry in `pages.json` (`name`, `hash`, and `nav` for a shell route or `server` for a server tab). Add diff --git a/tools/ui-smoke/baselines/admin-servers.png b/tools/ui-smoke/baselines/admin-servers.png new file mode 100644 index 0000000000..63916d0099 Binary files /dev/null and b/tools/ui-smoke/baselines/admin-servers.png differ diff --git a/tools/ui-smoke/baselines/fleet.png b/tools/ui-smoke/baselines/fleet.png new file mode 100644 index 0000000000..0590be3cf2 Binary files /dev/null and b/tools/ui-smoke/baselines/fleet.png differ diff --git a/tools/ui-smoke/baselines/ui-sql-overview.png b/tools/ui-smoke/baselines/ui-sql-overview.png new file mode 100644 index 0000000000..03431197bd Binary files /dev/null and b/tools/ui-smoke/baselines/ui-sql-overview.png differ diff --git a/tools/ui-smoke/baselines/ui-sql-queries.png b/tools/ui-smoke/baselines/ui-sql-queries.png new file mode 100644 index 0000000000..f75fe0a71f Binary files /dev/null and b/tools/ui-smoke/baselines/ui-sql-queries.png differ diff --git a/tools/ui-smoke/package-lock.json b/tools/ui-smoke/package-lock.json index efe771d757..d2376d36d3 100644 --- a/tools/ui-smoke/package-lock.json +++ b/tools/ui-smoke/package-lock.json @@ -8,12 +8,26 @@ "name": "darling-ui-smoke", "version": "0.0.0", "dependencies": { - "playwright-core": "1.64.0" + "pixelmatch": "7.1.0", + "playwright-core": "1.64.0", + "pngjs": "7.0.0" }, "engines": { "node": ">=20" } }, + "node_modules/pixelmatch": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-7.1.0.tgz", + "integrity": "sha512-1wrVzJ2STrpmONHKBy228LM1b84msXDUoAzVEl0R8Mz4Ce6EPr+IVtxm8+yvrqLYMHswREkjYFaMxnyGnaY3Ng==", + "license": "ISC", + "dependencies": { + "pngjs": "^7.0.0" + }, + "bin": { + "pixelmatch": "bin/pixelmatch" + } + }, "node_modules/playwright-core": { "version": "1.64.0", "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.64.0.tgz", @@ -25,6 +39,15 @@ "engines": { "node": ">=20" } + }, + "node_modules/pngjs": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-7.0.0.tgz", + "integrity": "sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==", + "license": "MIT", + "engines": { + "node": ">=14.19.0" + } } } } diff --git a/tools/ui-smoke/package.json b/tools/ui-smoke/package.json index 965d979395..278e61a8da 100644 --- a/tools/ui-smoke/package.json +++ b/tools/ui-smoke/package.json @@ -4,8 +4,12 @@ "private": true, "description": "Web UI smoke check for the Darling viewer pages and Web Admin (#5701). Not shipped.", "type": "module", - "engines": { "node": ">=20" }, + "engines": { + "node": ">=20" + }, "dependencies": { - "playwright-core": "1.64.0" + "pixelmatch": "7.1.0", + "playwright-core": "1.64.0", + "pngjs": "7.0.0" } } diff --git a/tools/ui-smoke/pages.json b/tools/ui-smoke/pages.json index f0dcb6667d..37f7704431 100644 --- a/tools/ui-smoke/pages.json +++ b/tools/ui-smoke/pages.json @@ -2,8 +2,12 @@ "_comment": [ "Pages the web UI smoke check loads (#5701). Every page gets the same checks (smoke.mjs): no console error, page error, failed request or /api/* answer >= 400; #main fills; the sidebar lists the seeded servers; the nav link for the route is active; no .strip.error or [role=alert]; no panel still loading or in error.", "Per page, 'expect' adds what the seed guarantees. A panel is found by its data-panel (the read name) or its data-title; data-panel is unambiguous only for a read that feeds one card, so key a fanout panel (several cards sharing one read) by title; 'state' is data (draws rows, a chart or tiles) or empty (shows its documented empty text); 'minRows' counts the rows of the first [data-grid] inside it. 'grids' does the same for the whole page (an Admin tab has no panel cards): 'minRows' or 'exactRows' on the Nth [data-grid] in #main.", + "screenshot: a page with this key also gets a baseline PNG (shots.mjs, #5744; tools/ui-smoke/baselines/.png). Its mask is a list of CSS selectors painted over before the compare, added to screenshotDefaults.mask (the status bar clock, the refresh hint, a chart tooltip): put in it anything whose text changes from run to run.", "allowList: a console or request message that is known and harmless. Each entry needs an 'issue' number and a 'match' substring. It starts empty." ], + "screenshotDefaults": { + "mask": ["#statusbar .sb-updated", "#refresh-hint", ".chart-tooltip"] + }, "allowList": [], "servers": [ "ui-sql", @@ -13,12 +17,14 @@ { "name": "fleet", "hash": "#/fleet", + "screenshot": { "mask": [".meta"] }, "nav": "fleet", "heading": "Fleet Overview" }, { "name": "ui-sql-overview", "hash": "#/server/ui-sql", + "screenshot": { "mask": [] }, "server": "ui-sql", "expect": { "panels": [ @@ -76,6 +82,7 @@ { "name": "ui-sql-queries", "hash": "#/server/ui-sql/queries", + "screenshot": { "mask": [] }, "server": "ui-sql", "expect": { "panels": [ @@ -90,6 +97,7 @@ { "name": "admin-servers", "hash": "#/admin/servers", + "screenshot": { "mask": [] }, "nav": "admin", "expect": { "grids": [ diff --git a/tools/ui-smoke/run-ui-smoke.ps1 b/tools/ui-smoke/run-ui-smoke.ps1 index f7ab61ff60..513ce5c635 100644 --- a/tools/ui-smoke/run-ui-smoke.ps1 +++ b/tools/ui-smoke/run-ui-smoke.ps1 @@ -21,6 +21,11 @@ .PARAMETER WorkDir Scratch folder for the config, logs, cluster and screenshots. Made if missing. .PARAMETER NoBuild Skip dotnet build (the caller already built the Release service). .PARAMETER Channel Browser channel for playwright-core: chrome (the runner's) or msedge. +.PARAMETER UpdateBaselines + Screenshot step (#5744): render the pages marked "screenshot" in pages.json and write new PNGs to + \out\new-baselines, comparing nothing. Without it the step compares with tools/ui-smoke/baselines. + Baselines come from the CI image only (the workflow's update_baselines input), never from a laptop. +.PARAMETER NoShots Skip the screenshot step (the smoke walk still runs). .PARAMETER SeedOnly Stop after seed.sql is applied: no npm, no browser walk. The WPF smoke check (#5745, wpf-smoke.yml) uses this to get a migrated, seeded store for the Darling viewer; the service is stopped, and the caller keeps the cluster and the config in -WorkDir\cfg\darling.json (the viewer reads it through DARLING_CONFIG). @@ -33,7 +38,9 @@ param( [string] $WorkDir = (Join-Path ([System.IO.Path]::GetTempPath()) 'darling-ui-smoke'), [switch] $NoBuild, [string] $Channel = 'chrome', - [switch] $SeedOnly + [switch] $SeedOnly, + [switch] $UpdateBaselines, + [switch] $NoShots ) $ErrorActionPreference = 'Stop' @@ -93,7 +100,10 @@ try { } while ((Get-Date) -lt $regDeadline) if (-not $registered) { throw "the service did not register ui-sql in the store in 120 s (migrations still running? see service.out.log)" } - Invoke-Psql @('-f', (Join-Path $here 'seed.sql')) + # One anchor for every seeded time, taken here so the screenshot step (smoke.mjs, shots.mjs) knows it: the store's clock, UTC, to the minute. + $anchor = (Invoke-Psql @('-c', "SELECT to_char(date_trunc('minute', now() AT TIME ZONE 'UTC'), 'YYYY-MM-DD HH24:MI:SS')") | Select-Object -First 1).Trim() + if ($anchor -notmatch '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$') { throw "unexpected seed anchor: '$anchor'" } + Invoke-Psql @('-v', "anchor=$anchor", '-f', (Join-Path $here 'seed.sql')) if ($SeedOnly) { Write-Host "store seeded; config is $cfgPath" @@ -107,6 +117,13 @@ try { $env:UI_SMOKE_URL = $base $env:UI_SMOKE_OUT = Join-Path $WorkDir 'out' $env:UI_SMOKE_CHANNEL = $Channel + $env:UI_SMOKE_ANCHOR = $anchor + $env:UI_SMOKE_SHOTS = if ($NoShots) { 'off' } elseif ($UpdateBaselines) { 'update' } else { 'compare' } + if (-not $NoShots) { + # The screenshot pages use Playwright's own Chromium, pinned by the playwright-core version (the runner's Chrome updates with the image). + & npx --no-install playwright-core install chromium + if ($LASTEXITCODE -ne 0) { throw "playwright install chromium failed ($LASTEXITCODE)" } + } & node smoke.mjs $smokeExit = $LASTEXITCODE } diff --git a/tools/ui-smoke/seed.sql b/tools/ui-smoke/seed.sql index bc19093d19..7644c10a1d 100644 --- a/tools/ui-smoke/seed.sql +++ b/tools/ui-smoke/seed.sql @@ -4,7 +4,8 @@ -- psql -h 127.0.0.1 -p -U darling -d darling -v ON_ERROR_STOP=1 -f seed.sql -- -- Rules, so the assertions in pages.json can be exact: --- * Every time is relative to the store's own clock: now() AT TIME ZONE 'UTC', cut to the minute. Nothing is dated. +-- * Every time is relative to ONE anchor: the store's own clock in UTC, cut to the minute, or the psql variable `anchor` when the +-- caller sets it (-v anchor=...; run-ui-smoke.ps1 does, so the screenshot step knows it, #5744). Nothing is dated. -- * Every count is fixed: the same rows on every run, whatever the minute. -- * The targets are the two servers the config registers: "ui-sql" (SQL Server) and "ui-pg" (PostgreSQL), both at an -- address nothing listens on, so the service never collects and never overwrites what is seeded here. @@ -22,6 +23,12 @@ -- pg_statement_text 6 | collection_log 3 collectors x 72 \set ON_ERROR_STOP on +-- The anchor, as UTC text. Set by the caller (-v anchor='2026-01-01 12:00:00'), else taken once here, from the store's clock. +\if :{?anchor} +\else +SELECT to_char(date_trunc('minute', now() AT TIME ZONE 'UTC'), 'YYYY-MM-DD HH24:MI:SS') AS anchor +\gset +\endif BEGIN; -- The registry row the config seed made. server_name in every collect table is the STORAGE name (the host, for a SQL @@ -34,14 +41,14 @@ WHERE name = 'ui-sql' -- The observed-registry row (the service writes it on a first connect, which never happens here). INSERT INTO collect.servers (server_id, server_name, display_name, is_enabled, sql_engine_edition, sql_major_version, engine_kind, created_date, modified_date) VALUES (:sid, :'sname', 'ui-sql', TRUE, 3, 16, 'sqlserver', - (now() AT TIME ZONE 'UTC') - interval '7 days', now() AT TIME ZONE 'UTC') + (:'anchor'::timestamp) - interval '7 days', :'anchor'::timestamp) ON CONFLICT (server_id) DO UPDATE SET server_name = EXCLUDED.server_name, display_name = EXCLUDED.display_name, is_enabled = TRUE, sql_engine_edition = 3, sql_major_version = 16, engine_kind = 'sqlserver', modified_date = EXCLUDED.modified_date; -- The minutes: m = 0 is the current minute, m = 359 is six hours earlier. CREATE TEMP TABLE ui_minutes ON COMMIT DROP AS -SELECT m, date_trunc('minute', now() AT TIME ZONE 'UTC') - make_interval(mins => m) AS t +SELECT m, :'anchor'::timestamp - make_interval(mins => m) AS t FROM generate_series(0, 359) AS m; -- cpu_utilization_stats: Overview and CPU tabs. collection_id 1000000 + m. @@ -94,7 +101,7 @@ SELECT 5000000 + 1000 * q.n + c.c, c.t, :sid, :'sname', q.db, '0x' || lpad(to_he q.cpu_us / 2, q.cpu_us * 3, q.cpu_us, q.cpu_us * 6, q.query_text, '0x02' || lpad(to_hex(q.n), 8, '0'), '0x06' || lpad(to_hex(q.n), 8, '0'), q.execs, q.execs * q.cpu_us, q.execs * q.cpu_us * 2, q.execs * q.reads, q.execs / 10, q.execs / 50, q.execs * 3, 600 -FROM (SELECT cc AS c, date_trunc('minute', now() AT TIME ZONE 'UTC') - make_interval(mins => cc * 10) AS t FROM generate_series(0, 35) AS cc) c +FROM (SELECT cc AS c, :'anchor'::timestamp - make_interval(mins => cc * 10) AS t FROM generate_series(0, 35) AS cc) c CROSS JOIN (VALUES (1, 'SalesDb', 'SELECT TOP (100) o.OrderId, o.Total FROM dbo.Orders AS o WHERE o.CustomerId = @CustomerId ORDER BY o.OrderDate DESC;', 600, 4200, 900), (2, 'SalesDb', 'UPDATE dbo.Orders SET Status = @Status WHERE OrderId = @OrderId;', 900, 800, 120), @@ -130,7 +137,7 @@ CROSS JOIN (VALUES (1, 'ROWS', '.mdf', 'D:\data\', 20480), (2, 'LOG', '_log.ldf' -- collection_log: Health and Collection Health. 5 collectors, every 5 minutes, all success. log_id 8000000 + 1000 * collector + k. INSERT INTO collect.collection_log (log_id, server_id, server_name, collector_name, collection_time, duration_ms, status, rows_collected) SELECT 8000000 + 1000 * cl.n + k.k, :sid, :'sname', cl.name, - date_trunc('minute', now() AT TIME ZONE 'UTC') - make_interval(mins => k.k * 5), 120 + k.k, 'SUCCESS', cl.rows_collected + :'anchor'::timestamp - make_interval(mins => k.k * 5), 120 + k.k, 'SUCCESS', cl.rows_collected FROM generate_series(0, 71) AS k(k) CROSS JOIN (VALUES (1, 'wait_stats', 4), (2, 'cpu_utilization_stats', 1), (3, 'memory_stats', 1), (4, 'file_io_stats', 2), (5, 'query_stats', 12)) AS cl(n, name, rows_collected); @@ -206,7 +213,7 @@ WHERE name = 'ui-pg' INSERT INTO collect.servers (server_id, server_name, display_name, is_enabled, engine_kind, postgres_major_version, created_date, modified_date) VALUES (:pgid, :'pgname', 'ui-pg', TRUE, 'postgres', 16, - (now() AT TIME ZONE 'UTC') - interval '7 days', now() AT TIME ZONE 'UTC') + (:'anchor'::timestamp) - interval '7 days', :'anchor'::timestamp) ON CONFLICT (server_id) DO UPDATE SET server_name = EXCLUDED.server_name, display_name = EXCLUDED.display_name, is_enabled = TRUE, engine_kind = 'postgres', postgres_major_version = 16, modified_date = EXCLUDED.modified_date; @@ -235,7 +242,7 @@ CROSS JOIN (VALUES (1, 'IO', 'DataFileRead', 50, 400000), (2, 'Lock', 'transacti -- collections each (every 10 minutes); the read groups on (queryid, database_id), so the grid has exactly 6 rows. -- collection_id 22000000 + 1000 * query + c. INSERT INTO collect.pg_statement_text (server_id, queryid, query_text, first_seen, last_seen) -SELECT :pgid, q.n, q.query_text, (now() AT TIME ZONE 'UTC') - interval '2 days', now() AT TIME ZONE 'UTC' +SELECT :pgid, q.n, q.query_text, (:'anchor'::timestamp) - interval '2 days', :'anchor'::timestamp FROM (VALUES (1, 'SELECT id, total FROM orders WHERE customer_id = $1 ORDER BY created_at DESC LIMIT 100'), (2, 'UPDATE orders SET status = $1 WHERE id = $2'), @@ -253,13 +260,13 @@ SELECT 22000000 + 1000 * q.n + c.c, c.t, :pgid, :'pgname', q.n, CASE WHEN q.n <= (36 - c.c) * q.calls * 40, (36 - c.c) * q.calls, (36 - c.c) * q.calls / 20, (36 - c.c) * q.calls / 40, 0, (36 - c.c) * q.calls / 100, (36 - c.c) * q.calls / 10, (36 - c.c) * q.calls / 500, (36 - c.c) * q.calls * 64, q.calls, q.calls * q.ms, q.calls * 3, 600 -FROM (SELECT cc AS c, date_trunc('minute', now() AT TIME ZONE 'UTC') - make_interval(mins => cc * 10) AS t FROM generate_series(0, 35) AS cc) c +FROM (SELECT cc AS c, :'anchor'::timestamp - make_interval(mins => cc * 10) AS t FROM generate_series(0, 35) AS cc) c CROSS JOIN (VALUES (1, 800, 3), (2, 1200, 2), (3, 40, 250), (4, 1500, 1), (5, 90, 120), (6, 2, 4000)) AS q(n, calls, ms); -- collection_log (ui-pg): 3 collectors, every 5 minutes, all success. log_id 23000000 + 1000 * collector + k. INSERT INTO collect.collection_log (log_id, server_id, server_name, collector_name, collection_time, duration_ms, status, rows_collected) SELECT 23000000 + 1000 * cl.n + k.k, :pgid, :'pgname', cl.name, - date_trunc('minute', now() AT TIME ZONE 'UTC') - make_interval(mins => k.k * 5), 90 + k.k, 'SUCCESS', cl.rows_collected + :'anchor'::timestamp - make_interval(mins => k.k * 5), 90 + k.k, 'SUCCESS', cl.rows_collected FROM generate_series(0, 71) AS k(k) CROSS JOIN (VALUES (1, 'pg_database_stats', 2), (2, 'pg_wait_stats', 3), (3, 'pg_statement_stats', 6)) AS cl(n, name, rows_collected); diff --git a/tools/ui-smoke/shots.mjs b/tools/ui-smoke/shots.mjs new file mode 100644 index 0000000000..c03195ca10 --- /dev/null +++ b/tools/ui-smoke/shots.mjs @@ -0,0 +1,245 @@ +// Screenshot baselines for the pages marked "screenshot" in pages.json (#5744, phase 2 of #5701). +// +// playwright-core has no toHaveScreenshot (that is @playwright/test's runner), so this does the same job in a few lines: +// page.screenshot() with animations off and the caret hidden, retaken until two shots in a row are identical (the stability +// rule toHaveScreenshot uses), then compared with pixelmatch against tools/ui-smoke/baselines/.png. +// +// UI_SMOKE_SHOTS "compare" (default) | "update" (write new PNGs, compare nothing) | "off" +// UI_SMOKE_ANCHOR the seed's anchor, UTC, "YYYY-MM-DD HH:MM:SS" (run-ui-smoke.ps1 passes the one it gave seed.sql) +// UI_SMOKE_BASELINES folder of baselines (default ./baselines) +// +// Why the page looks the same on every run. The seed dates every row from its anchor (the minute the seed ran), and the service +// reads "the last N hours" from its own clock, so left alone the axis labels and every timestamp move with the minute. The +// service must keep its real clock (as_of is refused past its reach, so a seed dated in the past would rot), so the BROWSER is +// moved instead: every /api answer has its timestamps shifted by (FIXED_ANCHOR - seed anchor), every windowed read is asked with +// as_of = seed anchor + 1 minute so the window never drifts with how long the walk took, and the page clock is pinned to +// FIXED_ANCHOR + 1 minute. The page then shows a store seeded at FIXED_ANCHOR, whatever minute the run really started. +import { chromium } from "playwright-core"; +import { readFileSync, writeFileSync, mkdirSync, existsSync, copyFileSync, rmSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { PNG } from "pngjs"; +import pixelmatch from "pixelmatch"; + +export const FIXED_ANCHOR_MS = Date.UTC(2026, 2, 15, 12, 0, 0); +const AS_OF_AFTER_ANCHOR_MS = 60000; +/* Zero differing pixels, like toHaveScreenshot. Measured (#5744): unedited pages render 0 pixels apart run to run, and a one-label Fleet change is only 238 pixels (0.023%), which a 1% limit let through. */ +const MAX_DIFF_PIXELS = 0; +const PIXEL_THRESHOLD = 0.1; +const MAX_SHOT_TRIES = 8; +/* How long to wait for every card to turn collection stale. The band opens 2 minutes after the seed anchor (the run log prints the wait it took). */ +const STALE_WAIT_MS = 4 * 60 * 1000; +/* Screenshot-only: the status bar's database size is not the same on every run. Its text changes width when it crosses a digit count + (99 MB to 100 MB), a mask hides the glyphs but not the box, and in a CI run (38437783077) the item was not there at all, because the + size read had not finished. So the item and the separator before it are hidden: the bar is the same with or without it. */ +const SHOT_STYLE = "#statusbar .sb-store-size, #statusbar .sb-sep:has(+ .sb-store-size) { display: none !important; }"; +/* A timestamp farther than this from the seed anchor is not seed-relative (a build date, a plan's compile time): left alone. */ +const SHIFT_REACH_MS = 60 * 24 * 3600 * 1000; +const TS = /(\d{4})-(\d{2})-(\d{2})([T ])(\d{2}):(\d{2}):(\d{2})(\.\d+)?(Z|[+-]\d{2}:?\d{2})?/g; + +const pad = (n, w = 2) => String(n).padStart(w, "0"); + +/** Shift every ISO timestamp in `text` that lies near `anchorMs` by `deltaMs`, keeping its shape (T or space, fraction, offset). */ +export function shiftTimestamps(text, anchorMs, deltaMs) { + let shifted = 0; + const out = text.replace(TS, (m, y, mo, d, sep, h, mi, s, frac, tz) => { + let offMin = 0; + if (tz && tz !== "Z") { + const sign = tz[0] === "-" ? -1 : 1; + const digits = tz.slice(1).replace(":", ""); + offMin = sign * (Number(digits.slice(0, 2)) * 60 + Number(digits.slice(2))); + } + const instant = Date.UTC(Number(y), Number(mo) - 1, Number(d), Number(h), Number(mi), Number(s)) - offMin * 60000; + if (Math.abs(instant - anchorMs) > SHIFT_REACH_MS) return m; + const local = new Date(instant + deltaMs + offMin * 60000); + shifted++; + return ( + `${local.getUTCFullYear()}-${pad(local.getUTCMonth() + 1)}-${pad(local.getUTCDate())}${sep}` + + `${pad(local.getUTCHours())}:${pad(local.getUTCMinutes())}:${pad(local.getUTCSeconds())}${frac || ""}${tz || ""}` + ); + }); + return { text: out, shifted }; +} + +/** "2026-03-15 12:00:00" (UTC) to epoch ms. */ +export function parseAnchor(text) { + const m = /^(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2}):(\d{2})/.exec(String(text || "").trim()); + if (!m) return NaN; + return Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3]), Number(m[4]), Number(m[5]), Number(m[6])); +} + +/* toHaveScreenshot's rule: shoot until two in a row are byte-identical, up to a bound, else say it never settled. */ +async function stableShot(page, masks) { + let last = null; + for (let i = 0; i < MAX_SHOT_TRIES; i++) { + const png = await page.screenshot({ + type: "png", + mask: masks, + maskColor: "#ff00ff", + animations: "disabled", + caret: "hide", + scale: "css", + }); + if (last && Buffer.compare(last, png) === 0) return png; + last = png; + await page.waitForTimeout(250); + } + throw new Error(`the page never settled: ${MAX_SHOT_TRIES} screenshots in a row, no two identical`); +} + +async function shoot(browser, spec, ctx) { + const { baseUrl, anchorMs, deltaMs, settle, config } = ctx; + const context = await browser.newContext({ + viewport: { width: 1280, height: 800 }, + deviceScaleFactor: 1, + colorScheme: "light", + reducedMotion: "reduce", + locale: "en-US", + timezoneId: "UTC", + bypassCSP: true, + }); + const page = await context.newPage(); + const asOf = new Date(anchorMs + AS_OF_AFTER_ANCHOR_MS).toISOString(); + let shiftedCount = 0; + await context.route("**/api/**", async (route) => { + const url = new URL(route.request().url()); + if (route.request().method() !== "GET") return route.continue(); + if (url.pathname.startsWith("/api/read/") && url.searchParams.has("hours") && !url.searchParams.has("as_of")) url.searchParams.set("as_of", asOf); + const response = await route.fetch({ url: url.toString() }); + const type = response.headers()["content-type"] || ""; + if (!type.includes("json")) return route.fulfill({ response }); + const r = shiftTimestamps(await response.text(), anchorMs, deltaMs); + shiftedCount += r.shifted; + return route.fulfill({ response, body: r.text }); + }); + /* Pinned after the shift above has been agreed: a minute after the fixed anchor, as the real clock is a minute after the seed's. */ + await page.clock.setFixedTime(new Date(FIXED_ANCHOR_MS + AS_OF_AFTER_ANCHOR_MS)); + try { + await page.goto(baseUrl + "/" + spec.hash, { waitUntil: "domcontentloaded" }); + /* A stylesheet, not an inline style: the status bar is re-mounted on a refresh. The page's CSP (style-src 'self') is why the context bypasses it. */ + await page.addStyleTag({ content: SHOT_STYLE }); + await settle(page); + await page.evaluate(() => document.fonts.ready); + const selectors = [...(config.screenshotDefaults?.mask || []), ...(spec.screenshot?.mask || [])]; + const masks = selectors.map((s) => page.locator(s)); + const png = await stableShot(page, masks); + return { png, shifted: shiftedCount }; + } finally { + await context.close(); + } +} + +/** Run the screenshot pages. Returns the problems (strings); writes PNGs and, on a mismatch, the baseline/actual/diff triple. */ +export async function runShots({ config, baseUrl, outDir, only, settle }) { + const mode = (process.env.UI_SMOKE_SHOTS || "compare").toLowerCase(); + if (mode === "off") return []; + const specs = config.pages.filter((p) => p.screenshot && (!only.length || only.includes(p.name))); + if (specs.length === 0) return []; + const anchorMs = parseAnchor(process.env.UI_SMOKE_ANCHOR); + if (Number.isNaN(anchorMs)) return ["screenshots: UI_SMOKE_ANCHOR (the seed's anchor, UTC) is not set; run through run-ui-smoke.ps1"]; + const baselines = process.env.UI_SMOKE_BASELINES || join(dirname(fileURLToPath(import.meta.url)), "baselines"); + const newDir = join(outDir, "new-baselines"); + const failDir = join(outDir, "shot-failures"); + mkdirSync(newDir, { recursive: true }); + mkdirSync(failDir, { recursive: true }); + + /* The seed's anchor is the minute it ran; the service answers as_of only up to its own clock, so wait out the minute. */ + const waitMs = anchorMs + AS_OF_AFTER_ANCHOR_MS + 2000 - Date.now(); + if (waitMs > 0) await new Promise((r) => setTimeout(r, waitMs)); + + /* The service marks a server it cannot reach "collection stale" some minutes after it started, on its own clock: a page shot before + that shows "Online", one shot after shows "Warning" (the red-proof run caught this, #5744). Wait until /api/fleet says every + card is stale, so every run photographs the same state. + "Stale" is a BAND, not a threshold: it opens 2 minutes after a server's last collection and closes at 30 (ServerHealthBands.cs, + where the card turns Offline and is_online goes false). So the wait has two ways to fail, and each names its own cause: the cards + are already past the band (the seed anchor aged past 30 minutes before the shots started), or they never reached it. The anchor + is a minute old when this starts and the band opens at 2 minutes, so a real run is stale within a minute or two; the deadline is + a few minutes, well inside the job's timeout-minutes. */ + const staleStarted = Date.now(); + const staleDeadline = staleStarted + STALE_WAIT_MS; + let seen = "no answer from /api/fleet"; + for (;;) { + try { + const fleet = await (await fetch(baseUrl + "/api/fleet")).json(); + const cards = fleet.cards || []; + const stale = cards.filter((c) => c.collection_stale === true).length; + const offline = cards.filter((c) => c.is_online === false).length; + seen = `${cards.length} cards, ${stale} stale, ${offline} offline`; + if (cards.length > 0 && stale === cards.length) { + console.log(`ok fleet is collection stale after ${((Date.now() - staleStarted) / 1000).toFixed(0)}s of waiting (${seen})`); + break; + } + if (offline > 0) { + return [`screenshots: the fleet is already past collection stale (${seen}); the cards go Offline 30 minutes after the seed anchor, so the walk ran too long for the seed to stay stale. Shorten the walk or re-seed.`]; + } + } catch { + /* the service answers again on the next poll */ + } + if (Date.now() > staleDeadline) { + return [`screenshots: the fleet never reached collection stale in ${STALE_WAIT_MS / 60000} minutes (${seen}); the band opens 2 minutes after the seed anchor, so the service clock or the seed anchor is wrong, or /api/fleet stopped answering`]; + } + await new Promise((r) => setTimeout(r, 5000)); + } + + /* Playwright's own Chromium, pinned by the playwright-core version (the workflow installs and caches it): the runner's Chrome + updates with the image and would move pixels. */ + const browser = await chromium.launch({ headless: true }); + const problems = []; + const ctx = { baseUrl, anchorMs, deltaMs: FIXED_ANCHOR_MS - anchorMs, settle, config }; + try { + for (const spec of specs) { + let r; + try { + r = await shoot(browser, spec, ctx); + } catch (e) { + problems.push(`screenshot ${spec.name}: ${e && e.message ? e.message.split("\n")[0] : e}`); + console.log(`FAIL shot ${spec.name}: ${problems[problems.length - 1]}`); + continue; + } + if (mode === "update") { + writeFileSync(join(newDir, `${spec.name}.png`), r.png); + console.log(`ok shot ${spec.name}: new baseline written (${r.png.length} bytes, ${r.shifted} timestamps shifted)`); + continue; + } + const basePath = join(baselines, `${spec.name}.png`); + if (!existsSync(basePath)) { + writeFileSync(join(failDir, `${spec.name}.actual.png`), r.png); + problems.push(`screenshot ${spec.name}: no baseline at ${basePath}; dispatch the workflow with update_baselines and commit the PNGs`); + console.log(`FAIL shot ${spec.name}: no baseline`); + continue; + } + const base = PNG.sync.read(readFileSync(basePath)); + const actual = PNG.sync.read(r.png); + writeFileSync(join(failDir, `${spec.name}.actual.png`), r.png); + copyFileSync(basePath, join(failDir, `${spec.name}.baseline.png`)); + if (base.width !== actual.width || base.height !== actual.height) { + problems.push(`screenshot ${spec.name}: size ${actual.width}x${actual.height}, baseline ${base.width}x${base.height}`); + console.log(`FAIL shot ${spec.name}: size differs`); + continue; + } + const diff = new PNG({ width: base.width, height: base.height }); + const differing = pixelmatch(base.data, actual.data, diff.data, base.width, base.height, { threshold: PIXEL_THRESHOLD }); + const ratio = differing / (base.width * base.height); + if (differing > MAX_DIFF_PIXELS) { + writeFileSync(join(failDir, `${spec.name}.diff.png`), PNG.sync.write(diff)); + problems.push(`screenshot ${spec.name}: ${differing} pixels differ (${(ratio * 100).toFixed(3)}%, none allowed); baseline, actual and diff are in shot-failures`); + console.log(`FAIL shot ${spec.name}: ${differing} pixels (${(ratio * 100).toFixed(3)}%) differ`); + } else { + /* A pass keeps no triple: the failure folder holds only what failed. */ + for (const suffix of ["actual", "baseline"]) rmQuiet(join(failDir, `${spec.name}.${suffix}.png`)); + console.log(`ok shot ${spec.name}: ${differing} pixels differ (${(ratio * 100).toFixed(3)}%), ${r.shifted} timestamps shifted`); + } + } + } finally { + await browser.close(); + } + return problems; +} + +function rmQuiet(p) { + try { + rmSync(p, { force: true }); + } catch { + /* nothing to remove */ + } +} diff --git a/tools/ui-smoke/smoke.mjs b/tools/ui-smoke/smoke.mjs index 66d80c30ec..ade2df0be0 100644 --- a/tools/ui-smoke/smoke.mjs +++ b/tools/ui-smoke/smoke.mjs @@ -1,8 +1,8 @@ // Web UI smoke check for the Darling viewer pages and Web Admin (#5701). // // Loads every page in pages.json in a real Chrome (playwright-core, the browser already on the machine) against a -// running service seeded by seed.sql, and fails when a page shows a problem. No screenshots are compared; a screenshot -// is taken only of a page that failed. Run it through run-ui-smoke.ps1, which starts the store and the service. +// running service seeded by seed.sql, and fails when a page shows a problem. A screenshot is kept of a page that failed, and the +// pages marked "screenshot" in pages.json are compared with their baselines (shots.mjs, #5744). Run it through run-ui-smoke.ps1, which starts the store and the service. // // UI_SMOKE_URL base address of the web host (default http://127.0.0.1:5153) // UI_SMOKE_OUT folder for report.json and failure screenshots (default ./out) @@ -20,6 +20,7 @@ import { chromium } from "playwright-core"; import { readFileSync, mkdirSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; +import { runShots } from "./shots.mjs"; const here = dirname(fileURLToPath(import.meta.url)); const baseUrl = (process.env.UI_SMOKE_URL || "http://127.0.0.1:5153").replace(/\/+$/, ""); @@ -215,11 +216,24 @@ for (const spec of config.pages) { } await browser.close(); -writeFileSync(join(outDir, "report.json"), JSON.stringify(results, null, 2)); +/* The walk's result lands first (report.json is { pages, screenshotProblems }), so nothing the screenshot stage does can lose it. */ +writeFileSync(join(outDir, "report.json"), JSON.stringify({ pages: results, screenshotProblems: [] }, null, 2)); + +/* Screenshot baselines (#5744): the pages marked "screenshot" in pages.json, in Playwright's own pinned Chromium. A throw out of the + stage (Chromium not installed, an unwritable folder) is a screenshot problem like any other: it fails the run, it does not crash it. */ +let shotProblems; +try { + shotProblems = await runShots({ config, baseUrl, outDir, only, settle }); +} catch (e) { + shotProblems = [`screenshots: the stage threw: ${e && e.message ? e.message.split("\n")[0] : e}`]; +} +for (const p of shotProblems) console.log(` PROBLEM ${p}`); + +writeFileSync(join(outDir, "report.json"), JSON.stringify({ pages: results, screenshotProblems: shotProblems }, null, 2)); const failed = results.filter((r) => r.problems.length); console.log(`\nui-smoke: ${results.length - failed.length}/${results.length} pages clean, ${results.filter((r) => r.flaked).length} flaked`); if (results.length === 0) { console.log("ui-smoke: no page ran."); process.exit(1); } -process.exit(failed.length ? 1 : 0); +process.exit(failed.length || shotProblems.length ? 1 : 0);