Web UI check: screenshot baselines for four Darling pages (#5744)

.gitattributes

@@ -32,3 +32,5 @@ Darling/Darling.Tests/Fixtures/ManagedConf/rehearsal-postgresql.conf text eol=lf
 tools/pg-verification-rig/Dockerfile text eol=lf
 tools/pg-verification-rig/*.yml text eol=lf
 tools/pg-verification-rig/*.sql text eol=lf
+# Screenshot baselines for the web UI check (#5744): PNGs, never diffed or converted as text.
+tools/ui-smoke/baselines/*.png binary

.github/workflows/darling-ui-smoke.yml

@@ -2,7 +2,8 @@ name: Darling UI smoke
 
 # #5701 phase 1: loads the Darling web pages and Web Admin in a real Chrome against a service on a throwaway,
 # seeded PostgreSQL store, and fails on a console error, a failed request, an error strip, or a panel or grid that
-# the seed says must have data and does not. No screenshots are compared. See tools/ui-smoke/ (run-ui-smoke.ps1 is
+# the seed says must have data and does not. #5744 (phase 2): four pages are also compared with screenshot baselines
+# (tools/ui-smoke/baselines/*.png, in Playwright's own pinned Chromium). See tools/ui-smoke/ (run-ui-smoke.ps1 is
 # the entry, locally and here).
 #
 # Triggers: a pull request touching the web files, the harness or this workflow (a draft pull request is skipped
@@ -20,7 +21,15 @@ on:
       - '.github/workflows/darling-ui-smoke.yml'
   # The nightly runs it as a reusable workflow (nightly.yml, outside publish's needs).
   workflow_call:
+  # update_baselines: render the four screenshot pages and upload new PNGs as the artifact "darling-ui-smoke-new-baselines"
+  # (nothing is compared and nothing is committed). Baselines come from this CI image only: download the artifact and commit
+  # the PNGs to tools/ui-smoke/baselines/ on the pull request, where GitHub's image diff is the review.
   workflow_dispatch:
+    inputs:
+      update_baselines:
+        description: 'Write new screenshot baselines (artifact darling-ui-smoke-new-baselines) instead of comparing'
+        type: boolean
+        default: false
 
 permissions:
   contents: read
@@ -35,8 +44,13 @@ jobs:
     # Skip a draft pull request; a dispatch has no pull request and always runs.
     if: github.event_name != 'pull_request' || github.event.pull_request.draft == false
     runs-on: windows-latest
-    # Expected 8 to 10 minutes warm (service build 2, PostgreSQL 1, browser walk 2 to 3); 3 more on a cold pg-runtime cache.
+    # Measured (#5744, five green runs with both caches warm): 3.4 to 5.8 minutes in all. A typical run: checkout and .NET 0.5, service build
+    # 1 to 1.2, PostgreSQL 0.2, browser walk and the stale wait 1.7, four screenshots 0.1. A cold pg-runtime cache adds 3. The 20 stays: over 3 times the slowest run.
     timeout-minutes: 20
+    # One explicit home for Playwright's Chromium. The `env` context holds only variables declared in an env: map (never the runner's
+    # own LOCALAPPDATA), so the cache step and the install in run-ui-smoke.ps1 both use this path, not Playwright's default.
+    env:
+      PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}\.playwright-browsers
     steps:
       - uses: actions/checkout@v7
 
@@ -101,10 +115,22 @@ jobs:
           & "$bin\createdb.exe" -h 127.0.0.1 -p 5541 -U darling darling
           if ($LASTEXITCODE -ne 0) { throw "createdb failed ($LASTEXITCODE)" }
 
+      # Playwright's own Chromium for the screenshot pages, pinned by the playwright-core version in package-lock.json and cached by it,
+      # at PLAYWRIGHT_BROWSERS_PATH (job env above), which `playwright-core install chromium` and chromium.launch() both honour.
+      - name: Cache Playwright Chromium
+        uses: actions/cache@v6
+        with:
+          path: ${{ env.PLAYWRIGHT_BROWSERS_PATH }}
+          key: playwright-chromium-${{ runner.os }}-${{ hashFiles('tools/ui-smoke/package-lock.json') }}
+
       - name: Run the web UI smoke check
         shell: pwsh
+        env:
+          UPDATE_BASELINES: ${{ github.event_name == 'workflow_dispatch' && inputs.update_baselines }}
         run: |
-          ./tools/ui-smoke/run-ui-smoke.ps1 -PgBin "$env:GITHUB_WORKSPACE\Darling\artifacts\pg-runtime\pgsql\bin" -PgPort 5541 -WorkDir "$env:RUNNER_TEMP\ui-smoke" -NoBuild -Channel chrome
+          $update = @{ }
+          if ($env:UPDATE_BASELINES -eq 'true') { $update.UpdateBaselines = $true }
+          ./tools/ui-smoke/run-ui-smoke.ps1 -PgBin "$env:GITHUB_WORKSPACE\Darling\artifacts\pg-runtime\pgsql\bin" -PgPort 5541 -WorkDir "$env:RUNNER_TEMP\ui-smoke" -NoBuild -Channel chrome @update
 
       - name: Logs tail
         if: always()
@@ -128,6 +154,26 @@ jobs:
           retention-days: 7
           if-no-files-found: ignore
 
+      # An update_baselines dispatch: the new PNGs, one per screenshot page, to download and commit.
+      - name: Upload new screenshot baselines
+        if: always() && github.event_name == 'workflow_dispatch' && inputs.update_baselines
+        uses: actions/upload-artifact@v6
+        with:
+          name: darling-ui-smoke-new-baselines
+          path: ${{ runner.temp }}\ui-smoke\out\new-baselines\*.png
+          retention-days: 14
+          if-no-files-found: warn
+
+      # A compare that failed: for each page that differs, the baseline, the new screenshot and the difference image.
+      - name: Upload screenshot differences
+        if: failure()
+        uses: actions/upload-artifact@v6
+        with:
+          name: darling-ui-smoke-screenshot-diffs
+          path: ${{ runner.temp }}\ui-smoke\out\shot-failures\*.png
+          retention-days: 14
+          if-no-files-found: ignore
+
       - name: Stop throwaway PostgreSQL
         if: always()
         shell: pwsh

tools/ui-smoke/README.md

@@ -2,8 +2,8 @@
 
 Loads every Darling web page and Web Admin tab in a real Chrome against a service running on a throwaway, seeded
 PostgreSQL store. It fails the job on a console error, a page error, a failed request, an answer of 400 or more, an
-error strip, or a panel or grid that the seed says must have data and does not. No screenshots are compared: a screenshot
-and a trace are kept only for a page that failed.
+error strip, or a panel or grid that the seed says must have data and does not. A screenshot and a trace are kept for a
+page that failed. Four pages are also compared with screenshot baselines (see "Screenshot baselines" below).
 
 It runs in `.github/workflows/darling-ui-smoke.yml` ("Darling web UI smoke") on a pull request that touches
 `Darling/PerformanceMonitor.Darling.Service/wwwroot/**`, `tools/ui-smoke/**` or the workflow (draft pull requests are
@@ -19,6 +19,8 @@ reports and does not stop publishing). It is not a required check: a path-filter
 | `seed.sql` | The rows. Every time is relative to the store's own clock and every count is fixed, so the assertions are exact. |
 | `pages.json` | The pages and what each must show. |
 | `smoke.mjs` | The browser walk (`playwright-core`, the installed Chrome or Edge). |
+| `shots.mjs` | The screenshot step (`playwright-core`'s own Chromium, `pixelmatch`, `pngjs`). |
+| `baselines/*.png` | The baseline screenshots, one per page marked `screenshot` in `pages.json`. |
 
 ## What it checks, per page
 
@@ -52,6 +54,46 @@ Logs, `report.json`, failure screenshots and traces land in `-WorkDir` (default
 trace with `npx playwright show-trace <page>.trace.zip`. `UI_SMOKE_PAGES=ui-pg-waits,alerts` runs only those pages when
 you call `node smoke.mjs` yourself.
 
+## Screenshot baselines
+
+Four pages have a baseline PNG in `baselines/`: `fleet`, `ui-sql-overview`, `ui-sql-queries` and `admin-servers`. After the
+smoke walk, `shots.mjs` renders each one at 1280 by 800 and compares it with its baseline. The check fails when any pixel
+differs by more than a small color threshold. The renders are the same from run to run, so there is no allowance. A page is marked by a `screenshot` key in `pages.json`.
+The key lists CSS selectors to paint over, for text that changes from run to run.
+
+How the page is kept the same on every run:
+
+- The browser is Playwright's own Chromium, pinned by the `playwright-core` version. It is not the runner's Chrome, which
+  updates with the runner image.
+- The viewport, scale (1), color scheme, locale (en-US), time zone (UTC) and reduced motion are fixed. Animations are off and
+  the caret is hidden.
+- The seed dates every row from one anchor, the minute `run-ui-smoke.ps1` ran it. The service must keep its real clock, so the
+  browser is moved instead. Every `/api` answer has its timestamps shifted to a fixed date. Every windowed read gets an
+  `as_of` of anchor plus one minute. The page clock is pinned to the same fixed date plus one minute.
+- A screenshot is taken again until two in a row are identical. If that does not happen in 8 tries, the page fails.
+- The service marks the two unreachable servers "collection stale" a couple of minutes after the seed anchor, and "offline"
+  after 30. The shots wait until `/api/fleet` shows every card stale. They give up after 4 minutes with one message if the
+  cards never got there, and with another if the cards are already past stale.
+- The database size in the status bar is not the same on every run. Its width changes when it gains a digit (99 MB to 100 MB).
+  In one run the item was missing, because the read of the size had not finished. A mask hides the text, not the box. For the
+  shots, `shots.mjs` adds a style that hides the item and the separator before it, so the bar is the same either way.
+- `report.json` is `{ pages, screenshotProblems }`. The page walk is written first, then rewritten with the screenshot
+  problems, so a failure in the screenshot step cannot lose the walk.
+- Playwright's Chromium lives in the folder named by `PLAYWRIGHT_BROWSERS_PATH`. The workflow sets it and caches that folder.
+
+To refresh the baselines (after a change to the page that you want to keep):
+
+1. Start the workflow "Darling UI smoke" by hand on your branch, with `update_baselines` set. For example:
+   `gh workflow run darling-ui-smoke.yml --ref <branch> -f update_baselines=true`.
+2. Download the artifact `darling-ui-smoke-new-baselines` from that run.
+3. Copy the PNGs into `tools/ui-smoke/baselines/` and commit them on the pull request.
+4. Review GitHub's image diff of each PNG. Never commit a PNG that you rendered on your own machine. Fonts and the browser
+   differ, so most pixels differ in CI.
+
+When a compare fails, the job uploads the artifact `darling-ui-smoke-screenshot-diffs`. For each page that differs, it holds
+`<page>.baseline.png`, `<page>.actual.png` and `<page>.diff.png` (the changed pixels). If the change is wanted, refresh the
+baselines as above. If a runner image update moved the pixels, refresh them the same way.
+
 ## Add a page or a seeded table
 
 - A page: one entry in `pages.json` (`name`, `hash`, and `nav` for a shell route or `server` for a server tab). Add

tools/ui-smoke/package.json

@@ -4,8 +4,12 @@
   "private": true,
   "description": "Web UI smoke check for the Darling viewer pages and Web Admin (#5701). Not shipped.",
   "type": "module",
-  "engines": { "node": ">=20" },
+  "engines": {
+    "node": ">=20"
+  },
   "dependencies": {
-    "playwright-core": "1.64.0"
+    "pixelmatch": "7.1.0",
+    "playwright-core": "1.64.0",
+    "pngjs": "7.0.0"
   }
 }

tools/ui-smoke/pages.json

@@ -2,8 +2,12 @@
   "_comment": [
     "Pages the web UI smoke check loads (#5701). Every page gets the same checks (smoke.mjs): no console error, page error, failed request or /api/* answer >= 400; #main fills; the sidebar lists the seeded servers; the nav link for the route is active; no .strip.error or [role=alert]; no panel still loading or in error.",
     "Per page, 'expect' adds what the seed guarantees. A panel is found by its data-panel (the read name) or its data-title; data-panel is unambiguous only for a read that feeds one card, so key a fanout panel (several cards sharing one read) by title; 'state' is data (draws rows, a chart or tiles) or empty (shows its documented empty text); 'minRows' counts the rows of the first [data-grid] inside it. 'grids' does the same for the whole page (an Admin tab has no panel cards): 'minRows' or 'exactRows' on the Nth [data-grid] in #main.",
+    "screenshot: a page with this key also gets a baseline PNG (shots.mjs, #5744; tools/ui-smoke/baselines/<name>.png). Its mask is a list of CSS selectors painted over before the compare, added to screenshotDefaults.mask (the status bar clock, the refresh hint, a chart tooltip): put in it anything whose text changes from run to run.",
     "allowList: a console or request message that is known and harmless. Each entry needs an 'issue' number and a 'match' substring. It starts empty."
   ],
+  "screenshotDefaults": {
+    "mask": ["#statusbar .sb-updated", "#refresh-hint", ".chart-tooltip"]
+  },
   "allowList": [],
   "servers": [
     "ui-sql",
@@ -13,12 +17,14 @@
     {
       "name": "fleet",
       "hash": "#/fleet",
+      "screenshot": { "mask": [".meta"] },
       "nav": "fleet",
       "heading": "Fleet Overview"
     },
     {
       "name": "ui-sql-overview",
       "hash": "#/server/ui-sql",
+      "screenshot": { "mask": [] },
       "server": "ui-sql",
       "expect": {
         "panels": [
@@ -76,6 +82,7 @@
     {
       "name": "ui-sql-queries",
       "hash": "#/server/ui-sql/queries",
+      "screenshot": { "mask": [] },
       "server": "ui-sql",
       "expect": {
         "panels": [
@@ -90,6 +97,7 @@
     {
       "name": "admin-servers",
       "hash": "#/admin/servers",
+      "screenshot": { "mask": [] },
       "nav": "admin",
       "expect": {
         "grids": [

tools/ui-smoke/run-ui-smoke.ps1

@@ -21,6 +21,11 @@
 .PARAMETER WorkDir     Scratch folder for the config, logs, cluster and screenshots. Made if missing.
 .PARAMETER NoBuild     Skip dotnet build (the caller already built the Release service).
 .PARAMETER Channel     Browser channel for playwright-core: chrome (the runner's) or msedge.
+.PARAMETER UpdateBaselines
+                       Screenshot step (#5744): render the pages marked "screenshot" in pages.json and write new PNGs to
+                       <WorkDir>\out\new-baselines, comparing nothing. Without it the step compares with tools/ui-smoke/baselines.
+                       Baselines come from the CI image only (the workflow's update_baselines input), never from a laptop.
+.PARAMETER NoShots     Skip the screenshot step (the smoke walk still runs).
 .PARAMETER SeedOnly    Stop after seed.sql is applied: no npm, no browser walk. The WPF smoke check (#5745, wpf-smoke.yml) uses this to
                        get a migrated, seeded store for the Darling viewer; the service is stopped, and the caller keeps the cluster
                        and the config in -WorkDir\cfg\darling.json (the viewer reads it through DARLING_CONFIG).
@@ -33,7 +38,9 @@ param(
     [string] $WorkDir = (Join-Path ([System.IO.Path]::GetTempPath()) 'darling-ui-smoke'),
     [switch] $NoBuild,
     [string] $Channel = 'chrome',
-    [switch] $SeedOnly
+    [switch] $SeedOnly,
+    [switch] $UpdateBaselines,
+    [switch] $NoShots
 )
 
 $ErrorActionPreference = 'Stop'
@@ -93,7 +100,10 @@ try {
     } while ((Get-Date) -lt $regDeadline)
     if (-not $registered) { throw "the service did not register ui-sql in the store in 120 s (migrations still running? see service.out.log)" }
 
-    Invoke-Psql @('-f', (Join-Path $here 'seed.sql'))
+    # One anchor for every seeded time, taken here so the screenshot step (smoke.mjs, shots.mjs) knows it: the store's clock, UTC, to the minute.
+    $anchor = (Invoke-Psql @('-c', "SELECT to_char(date_trunc('minute', now() AT TIME ZONE 'UTC'), 'YYYY-MM-DD HH24:MI:SS')") | Select-Object -First 1).Trim()
+    if ($anchor -notmatch '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$') { throw "unexpected seed anchor: '$anchor'" }
+    Invoke-Psql @('-v', "anchor=$anchor", '-f', (Join-Path $here 'seed.sql'))
 
     if ($SeedOnly) {
         Write-Host "store seeded; config is $cfgPath"
@@ -107,6 +117,13 @@ try {
         $env:UI_SMOKE_URL = $base
         $env:UI_SMOKE_OUT = Join-Path $WorkDir 'out'
         $env:UI_SMOKE_CHANNEL = $Channel
+        $env:UI_SMOKE_ANCHOR = $anchor
+        $env:UI_SMOKE_SHOTS = if ($NoShots) { 'off' } elseif ($UpdateBaselines) { 'update' } else { 'compare' }
+        if (-not $NoShots) {
+            # The screenshot pages use Playwright's own Chromium, pinned by the playwright-core version (the runner's Chrome updates with the image).
+            & npx --no-install playwright-core install chromium
+            if ($LASTEXITCODE -ne 0) { throw "playwright install chromium failed ($LASTEXITCODE)" }
+        }
         & node smoke.mjs
         $smokeExit = $LASTEXITCODE
     }