diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b59c5ad..fcf35b14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,8 +33,8 @@ auto-generated per-PR notes; this file is the curated, human-readable history. are untouched. Copy always copies the current detached result. The Table/JSON/Panel dispatch is now one shared `renderResultView` used by both the live pane and the detached view (no parallel copies). No new runtime - dependency. (Dashboard tiles keep their existing execution path for now; a - follow-up migrates them onto the shared streaming seam.) + dependency. (Dashboard tiles moved onto the same shared streaming seam in + #193 — see Changed below.) - **Schema column name and type are now independent drag targets** (#186). Dragging a column's name still inserts the SQL-safe quoted identifier; dragging its type meta now inserts the complete schema-provided ClickHouse @@ -95,6 +95,30 @@ auto-generated per-PR notes; this file is the curated, human-readable history. are preserved, never silently stripped. ### Changed +- **Dashboard tiles now stream through the shared `app.runReadInto` seam** (#193, + follow-up to #185). Every query-backed tile runs on the same execution path as + the workbench `run()` and the detached Data view instead of the bespoke + `queryDashboardTile` (`FORMAT JSON`, whole-response `parseJsonResult`) it used + before — gaining streaming transport, bounded client memory, live progress, and + **real per-tile cancellation** via an `AbortController`. The read-only guard is + preserved (`readonly:2` + `max_result_bytes` ride in the request; the row cap is + the `newResult('Table', DASH_TILE_ROW_CAP)` client trim against a server + `max_result_rows = CAP + 1` sentinel, so exactly-cap results are not flagged and + `>CAP` are trimmed and flagged). Each wave reserves its slot generation **at + creation** (aborting any in-flight request then) so a queued Refresh worker a + newer filter wave has superseded discards itself without issuing — closing a + stale-wave race; targeted filter re-runs now take one token preflight and the + same 6-way concurrency pool as full Refresh. While a tile streams, only its + loading placeholder's row count updates — panel classification and rendering + happen once, on completion, so charts are never rebuilt mid-stream. Two small, + deliberate behavior changes: a Dashboard panel query with an **explicit `FORMAT` + clause** is now rejected with a clear tile error (the streaming parser only + understands the structured stream, so a stray `FORMAT` would silently corrupt + the tile), and the tile footer always shows ms (wall-clock) and bytes (streamed + progress). The now-unused `app.runTile` / `queryDashboardTile` / + `dashboardTileSql` / `parseJsonResult` machinery is deleted so future cap or + settings fixes can't apply to only one path. No new runtime dependency; no change + to the workbench or detached view. - **One authoritative ClickHouse lexical scanner + structural lexer replaces the legacy highlighter tokenizer** (#182, supersedes #141). All string-based SQL analysis — statement splitting, parameter detection, optional-block/format diff --git a/src/core/dashboard.js b/src/core/dashboard.js index 54a21904..cc3313ff 100644 --- a/src/core/dashboard.js +++ b/src/core/dashboard.js @@ -1,12 +1,11 @@ // Pure logic for the Dashboard view (#149). No DOM, no globals. // // A dashboard is "the favorited subset of the Library, rendered together" — no -// new schema. This module holds the route helpers, the ClickHouse `FORMAT JSON` -// → array-rows transform the panel layer expects, and the tile result caps. +// new schema. This module holds the route helpers and the tile result caps. // (Per-tile classification moved to core/panel-cfg.js's autoPanel/resolvePanel -// in #166 — the panel union replaced classifyTile's chart-vs-skip ladder.) - -import { withTrailingFormat } from './format.js'; +// in #166 — the panel union replaced classifyTile's chart-vs-skip ladder. The +// tiles stream through the shared `app.runReadInto` seam as of #193, so the +// former `FORMAT JSON` → array-rows transform and its SQL prep were retired.) /** * True on the standalone dashboard route (a path ending in `/dashboard`, @@ -84,9 +83,10 @@ export function dashboardViewSelection(view) { /** * Rows kept per dashboard tile (#149 D9). Preserves the 5000-point line/area * chart cap (`CHART_ROW_CAPS` in `src/core/chart-data.js`) — a fetch cap below - * it would silently regress charts. `queryDashboardTile` requests - * `max_result_rows = cap + 1` (the `+1` is the truncation sentinel) and - * `parseJsonResult` trims to this bound client-side, which is the guarantee. + * it would silently regress charts. The tile streams with server + * `max_result_rows = cap + 1` (the `+1` is the truncation sentinel) while the + * client result's `newResult('Table', cap)` trims to `cap` and flags `capped` + * on the overshoot — the client-side trim is the guarantee (#193). */ export const DASH_TILE_ROW_CAP = 5000; @@ -106,50 +106,11 @@ export const DASH_TILE_BYTE_CAP = 50_000_000; */ export const DASH_TABLE_DISPLAY_CAP = 1000; -/** - * A favorite's SQL prepared for a one-shot tile fetch: `FORMAT JSON` appended - * unless the query already ends in its own trailing `FORMAT` clause (which we - * leave intact; a non-JSON format just errors the tile gracefully rather than - * being silently doubled). Delegates to `withTrailingFormat`, which strips a - * trailing `;`/comments and reuses `detectSqlFormat` (handling ClickHouse's - * `FORMAT x SETTINGS y` ordering). Empty input → '' (no favorite is empty). - */ -export function dashboardTileSql(sql) { - return withTrailingFormat(sql, 'JSON').sql; -} - -/** - * Transform a ClickHouse `FORMAT JSON` response into the shape the chart layer - * wants: `columns` = `meta` ([{name,type}]), `rows` = array-of-arrays (row[i] - * by column position), plus a small footer meta ({rows, ms, bytes, truncated}). - * - * `cap` (optional, #149 D9) is the guaranteed client-side row bound: when more - * than `cap` data rows arrive, `rows` is sliced to `cap` and `meta.truncated` - * is true. The server-side `max_result_rows = cap + 1` sentinel plus - * `result_overflow_mode:'break'` (see `queryDashboardTile`) overshoots at - * block boundaries, so the response's own `json.rows` is neither the full - * result count nor the displayed count — it is deliberately not exposed. - * `meta.rows` is the rows *shown* (`rows.length` after the trim); without a - * cap it is simply the row count, with `meta.truncated` false. - */ -export function parseJsonResult(json, cap) { - const columns = json.meta || []; - const data = json.data || []; - const truncated = cap != null && data.length > cap; - const rows = (truncated ? data.slice(0, cap) : data) - .map((o) => columns.map((c) => o[c.name])); - const stats = json.statistics || {}; - return { - columns, - rows, - meta: { - rows: rows.length, - ms: stats.elapsed != null ? Math.round(stats.elapsed * 1000) : null, - bytes: stats.bytes_read != null ? stats.bytes_read : null, - truncated, - }, - }; -} +// (The tiles' SQL prep + `FORMAT JSON` → array-rows transform — the former +// `dashboardTileSql` / `parseJsonResult` — were retired in #193 when the tiles +// moved onto the shared streaming `app.runReadInto` seam. The client row bound +// is now `newResult('Table', DASH_TILE_ROW_CAP)`'s trim + `capped` flag, and +// the tile result shape is pinned by `dashboardTileResult` in src/ui/dashboard.js.) // (The filter bar's field discovery moved to the parameter pipeline in #165: // `fieldControls(analysis)` in param-pipeline.js replaces the old diff --git a/src/net/ch-client.js b/src/net/ch-client.js index cdf82afe..c80e2d7a 100644 --- a/src/net/ch-client.js +++ b/src/net/ch-client.js @@ -10,7 +10,6 @@ import { parseExceptionText, isAuthExpiredBody, authDeniedMessage } from '../core/stream.js'; import { parseAstTables, buildSchemaGraph, externalDbs } from '../core/schema-graph.js'; import { sqlString } from '../core/format.js'; -import { DASH_TILE_ROW_CAP, DASH_TILE_BYTE_CAP } from '../core/dashboard.js'; /** Build a ClickHouse HTTP URL with query-string options. Pure. */ export function chUrl(origin, opts = {}) { @@ -91,33 +90,6 @@ export async function queryJson(ctx, sql, signal, extra, params) { return resp.json(); } -/** - * Run a favorite's SQL for a read-only dashboard tile (#149): `FORMAT JSON` plus - * the `readonly=2` HTTP setting, so a favorite that happens to contain a write - * (INSERT / ALTER / DROP / …) is rejected server-side rather than executed when - * the dashboard opens or refreshes — level 2 still permits SELECT and - * query-level `SETTINGS`. `params` (optional, #149 D3) forwards `param_` - * args for the dashboard's global filter bar. Returns parsed JSON; throws CH's - * reason on error. - * - * Result caps (#149 D9) are *best-effort* server hints: `max_result_rows` is - * `DASH_TILE_ROW_CAP + 1` — the `+1` is the truncation sentinel the client - * trim (`parseJsonResult(json, cap)`) detects — with `result_overflow_mode: - * 'break'` so an oversized result returns a truncated prefix (overshooting at - * block boundaries) instead of erroring, and `max_result_bytes` guards wide - * rows. Because level 2 still permits query-level `SETTINGS`, a favorite with - * `SETTINGS max_result_rows = 0` can override these URL settings — the - * client-side trim in `parseJsonResult` is the guaranteed bound. - */ -export function queryDashboardTile(ctx, sql, signal, params) { - return queryJson(ctx, sql, signal, { - readonly: 2, - max_result_rows: DASH_TILE_ROW_CAP + 1, - max_result_bytes: DASH_TILE_BYTE_CAP, - result_overflow_mode: 'break', - }, params); -} - /** * Run a `system.tables`/`system.columns` query (`sqlBody`, without its FORMAT * clause) with data-lake-catalog visibility enabled, falling back to the plain diff --git a/src/ui/app.js b/src/ui/app.js index 216903d2..70dd58ee 100644 --- a/src/ui/app.js +++ b/src/ui/app.js @@ -31,7 +31,7 @@ import { encodeShare } from '../core/share.js'; import { assembleReferenceData, buildCompletions } from '../core/completions.js'; import { generatePKCE, randomState } from '../core/pkce.js'; import { viewportZoom } from '../core/zoom-support.js'; -import { configBase, dashboardTileSql, parseJsonResult, DASH_TILE_ROW_CAP } from '../core/dashboard.js'; +import { configBase } from '../core/dashboard.js'; import { isQuerylessPanel } from '../core/panel-cfg.js'; import { snapshotAuth, restoreAuth, hasAuth, isAuthRequest, isAuthGrant, AUTH_REQUEST, AUTH_GRANT } from '../core/auth-handoff.js'; import * as oauthCfg from '../net/oauth-config.js'; @@ -2077,43 +2077,11 @@ export function createApp(env = {}) { } app.ensureFreshToken = ensureFreshToken; - // Run one favorite's SQL for a dashboard tile: read-only (writes rejected - // server-side by queryDashboardTile), FORMAT JSON, transformed to the - // array-row shape renderChart wants. `params` are the tile's prepared - // `param_` args from the dashboard's per-wave batch (#149 D3/#173) — - // the dashboard then passes `sql` already materialized (#165); when omitted - // (a standalone call) both are prepared here from the shared - // `state.varValues`/`state.filterActive`, mirroring the workbench's run(). Returns - // { columns, rows, meta } on success or { error } on failure. The token is - // resolved up front by ensureFreshToken (above), so this does not itself - // drive sign-out. - async function runTile(sql, params) { - try { - // ensureConfig + getToken are inside the try: getToken→refresh can THROW on - // a network/IdP failure, and a tile must degrade to { error } rather than - // reject (a rejected tile would break the whole grid's Promise.all). - // ensureConfig is memoized, so calling it here and in ensureFreshToken is - // cheap and keeps runTile usable on its own. - await ensureConfig(); - if (!(await getToken())) return { error: 'Not signed in' }; - let text = sql; - let args = params; - if (!args) { - const src = prepareTabSource(sql, wallNow()); - args = mergedSourceArgs(src); - // #165: swap in the materialized execution text only when the SQL - // actually is a template — block-free favorites keep their exact bytes. - if (hasOptionalBlocks(sql)) text = mergedSourceSql(src, sql); - } - const json = await ch.queryDashboardTile(chCtx, dashboardTileSql(text), undefined, args); - // DASH_TILE_ROW_CAP is the guaranteed client-side row bound (#149 D9): - // the server-side caps in queryDashboardTile are best-effort only. - return parseJsonResult(json, DASH_TILE_ROW_CAP); - } catch (e) { - return { error: String((e && e.message) || e) }; - } - } - app.runTile = runTile; + // Dashboard tiles stream their read-only SQL through the shared + // `app.runReadInto` seam directly (#193 — see src/ui/dashboard.js + // `runSlotTile`), the same path run() and the detached Data view use; the + // former bespoke `runTile`/`queryDashboardTile`/`parseJsonResult` machinery + // was retired so cap/settings fixes can't apply to only one path. app.renderDashboard = () => renderDashboard(app); // One-time cross-tab auth handoff. The dashboard opens in a new same-origin diff --git a/src/ui/dashboard.js b/src/ui/dashboard.js index 77e3a4d1..1c8c4ba3 100644 --- a/src/ui/dashboard.js +++ b/src/ui/dashboard.js @@ -3,8 +3,11 @@ // favorited Library query (a snapshot taken when the tab opens — Refresh // re-runs the data, it does not re-scan the Library). Favorites are // PARTITIONED BEFORE EXECUTION (#166): a text panel renders immediately with -// zero queries; everything else runs its SQL read-only via `app.runTile` and -// renders through the shared panel registry (panels.js) — an explicit saved +// zero queries; everything else streams its SQL read-only through the shared +// `app.runReadInto` seam (#193 — full streaming transport, server-side row cap, +// bounded client memory, and real per-tile AbortController cancellation, the +// same path the workbench run() and the detached Data view use) and renders +// through the shared panel registry (panels.js) — an explicit saved // `panel` wins (and never vanishes: zero-row explicit panels show an honest // "0 rows" state), an unconfigured result goes through the autoPanel // heuristic, and only unconfigured empty/single-row (future KPI) results are @@ -19,9 +22,11 @@ import { renderResolvedPanel } from './panels.js'; import { schemaKey } from '../core/chart-data.js'; import { resolvePanel, autoPanel } from '../core/panel-cfg.js'; import { - DASH_TILE_ROW_CAP, DASH_TABLE_DISPLAY_CAP, activeDashboardView, dashboardViewSelection, + DASH_TILE_ROW_CAP, DASH_TILE_BYTE_CAP, DASH_TABLE_DISPLAY_CAP, + activeDashboardView, dashboardViewSelection, } from '../core/dashboard.js'; -import { formatBytes, formatRows } from '../core/format.js'; +import { formatBytes, formatRows, detectSqlFormat } from '../core/format.js'; +import { newResult } from '../core/stream.js'; import { analyzeParameterizedSources, prepareParameterizedBatch, mergedSourceArgs, mergedSourceSql, fieldControls, } from '../core/param-pipeline.js'; @@ -61,15 +66,19 @@ function buildSeg(cls, options, getActive, onPick, ariaLabel) { } /** - * Build a tile's footer meta row (rows · ms · bytes), omitting stats CH didn't - * return. A fetch-truncated result (#149 D9: the client trimmed it to - * DASH_TILE_ROW_CAP) gets an honest note — client-side sort and chart - * aggregation only cover that fetched prefix, not the full underlying result. + * Build a tile's footer meta row (rows · ms · bytes). On the streaming seam + * (#193) `ms` is wall-clock (like run()'s finally) and `bytes` is the progress + * byte count — both always present — so the row is unconditional. A + * fetch-truncated result (#149 D9: the client trimmed it to DASH_TILE_ROW_CAP) + * gets an honest note — client-side sort and chart aggregation only cover that + * fetched prefix, not the full underlying result. */ function tileFooter(meta) { - const parts = [h('span', null, formatRows(meta.rows) + ' rows')]; - if (meta.ms != null) parts.push(h('span', null, meta.ms + ' ms')); - if (meta.bytes != null) parts.push(h('span', null, formatBytes(meta.bytes) + ' scanned')); + const parts = [ + h('span', null, formatRows(meta.rows) + ' rows'), + h('span', null, meta.ms + ' ms'), + h('span', null, formatBytes(meta.bytes) + ' scanned'), + ]; if (meta.truncated) { parts.push(h('span', null, 'first ' + DASH_TILE_ROW_CAP.toLocaleString() + ' rows fetched — sorting/charts cover this prefix only')); @@ -77,6 +86,29 @@ function tileFooter(meta) { return parts; } +/** + * Adapt a streamed `result` (from `app.runReadInto`) to the tile result shape + * `applyTileResult`/`tileFooter` expect (#193). `ms` is wall-clock (start→finish, + * like run()'s finally), `bytes` is the streamed progress byte count, and + * `truncated` reflects the client-side cap (`result.capped` — set once a row + * past `DASH_TILE_ROW_CAP` arrives). Only a successful, non-cancelled, + * current-generation result is ever applied (see runSlotTile). + */ +function dashboardTileResult(result, startedAt, finishedAt) { + return { + columns: result.columns, + rows: result.rows, + error: result.error, + cancelled: result.cancelled, + meta: { + rows: result.rows.length, + ms: Math.round(finishedAt - startedAt), + bytes: result.progress.bytes, + truncated: result.capped, + }, + }; +} + /** * Bounded-concurrency map that preserves append order. Workers grab the next * index in turn; each `worker` appends its card synchronously before its first @@ -104,10 +136,13 @@ async function runPool(items, limit, worker) { // updates this same slot's contents/visibility in place instead. `gen` is a // per-tile monotonically increasing generation counter guarding against // out-of-order responses (edit A, then B, before A's request returns — B's -// response must win); `destroy` tears down the slot's live panel instance -// (a chart's Chart.js object, via the registry's renderPanel contract) before -// it's replaced; `panelState` is the slot-persistent table-tile state (#166 — -// sort + column widths, keyed by result schema). +// response must win — and a queued Refresh worker that a newer wave has already +// superseded); `abortController` cancels this slot's in-flight streamed request +// when a newer wave supersedes it (#193); `destroy` tears down the slot's live +// panel instance (a chart's Chart.js object, via the registry's renderPanel +// contract) before it's replaced; `panelState` is the slot-persistent table-tile +// state (#166 — sort + column widths, keyed by result schema); `loadLabel` is +// the loading placeholder's live row-count text node (streamed progress, #193). function buildTileSlot(q) { const body = h('div', { class: 'dash-tile-body' }); const foot = h('div', { class: 'dash-tile-foot' }); @@ -117,7 +152,24 @@ function buildTileSlot(q) { h('span', { class: 'dash-tile-name', title: q.name }, q.name)); if (q.description) head.appendChild(h('div', { class: 'dash-tile-desc', title: q.description }, q.description)); const card = h('div', { class: 'dash-tile' }, head, body, foot); - return { card, body, foot, gen: 0, status: null, destroy: null, panelState: null }; + return { + card, body, foot, gen: 0, status: null, destroy: null, panelState: null, + abortController: null, loadLabel: null, + }; +} + +// Reserve the next generation for a slot AND abort its in-flight streamed +// request, atomically, at WAVE CREATION time (#193 design req 3). A queued +// Refresh worker only reaches its request when a pool slot frees up; reserving +// the generation up front (not when the worker starts) closes the stale-wave +// race where a slower older wave's worker finally runs a tile and supersedes a +// newer affected wave with older values. Returns the reserved generation; the +// worker re-checks `slot.gen === generation` before issuing and after streaming. +function supersedeSlot(slot) { + const generation = ++slot.gen; + if (slot.abortController) slot.abortController.abort(); + slot.abortController = null; + return generation; } function destroySlotChart(slot) { @@ -151,12 +203,19 @@ function renderTextSlot(app, q, slot) { function setSlotLoading(slot) { destroySlotChart(slot); slot.card.style.display = ''; - slot.body.replaceChildren(h('div', { class: 'dash-tile-load' }, Icon.spinner(), h('span', null, 'Loading…'))); + // Return the label node so streamed progress (onChunk, #193) can update just + // its text — "Loading… N rows" — without rebuilding the tile or classifying + // yet. Panel classification + rendering happen ONCE, after completion (never + // per chunk, which would thrash Chart.js and flash partial data). + const label = h('span', null, 'Loading…'); + slot.loadLabel = label; + slot.body.replaceChildren(h('div', { class: 'dash-tile-load' }, Icon.spinner(), label)); slot.foot.replaceChildren(); + return label; } // A tile whose SQL still has an empty/absent, or invalid (#170), {name:Type} -// value never calls app.runTile — it shows this placeholder instead (reusing +// value never issues a request — it shows this placeholder instead (reusing // the card's header/footer chrome so it doesn't look broken), and stays // visible: unlike a classifyTile `skip`, one filter value away it becomes // chartable, so it is NOT counted in the header's "N not shown" note. @@ -229,18 +288,23 @@ function applyTileResult(app, q, slot, r) { // Run (or re-run) one favorite's tile into its slot, gated by its prepared // source from the wave's batch (#173): unfilled OR invalid (#170) `{name:Type}` -// values show the placeholder (never calling app.runTile — an invalid value -// left to reach the server would either error confusingly or, for Int/UInt, -// silently wrap; see param-validate.js), a per-source error (e.g. a value -// that can't serialize for this tile's declaration) shows an error card — -// blocking only this tile, never its siblings — otherwise fetch with the -// batch's prepared args and classify. `onSettled()` fires after every -// transition (unfilled, errored or fetched) so the caller can recompute the -// live "N not shown" count. The generation bump happens before the gate check -// so a superseded in-flight fetch is discarded even if the newer edit resolves -// to "unfilled". -async function runSlotTile(app, q, slot, onSettled, src) { - const myGen = ++slot.gen; +// values show the placeholder (never issuing a request — an invalid value left +// to reach the server would either error confusingly or, for Int/UInt, silently +// wrap; see param-validate.js), a per-source error (e.g. a value that can't +// serialize for this tile's declaration) shows an error card — blocking only +// this tile, never its siblings — otherwise stream the SQL read-only through the +// shared `app.runReadInto` seam (#193) and classify ONCE on completion. +// `onSettled()` fires after every transition (unfilled, errored or fetched) so +// the caller can recompute the live "N not shown" count. +// +// `generation` was reserved (and any prior in-flight request aborted) by +// `supersedeSlot` at WAVE CREATION (#193 design req 3), not here: a queued +// Refresh worker whose slot a newer wave has already re-reserved discards itself +// up front without issuing, and a supersede mid-stream aborts this request and +// makes the post-await guard drop it — so a stale wave can never overwrite a +// newer one, even under the 6-way pool's queueing. +async function runSlotTile(app, q, slot, onSettled, src, generation) { + if (slot.gen !== generation) return; // a newer wave already superseded this queued tile if (src.missing.length || src.invalid.length) { setSlotUnfilled(slot, src.missing.concat(src.invalid)); onSettled(); @@ -251,18 +315,54 @@ async function runSlotTile(app, q, slot, onSettled, src) { onSettled(); return; } - setSlotLoading(slot); // The wire text is the wave's materialized execution view (#165) — only when // the favorite actually is a template; block-free SQL keeps its exact bytes. const execSql = hasOptionalBlocks(q.sql) ? mergedSourceSql(src, q.sql) : q.sql; - const r = await app.runTile(execSql, mergedSourceArgs(src)); - if (slot.gen !== myGen) return; // a newer edit started after this fetch; discard + // #193 design req 5: the shared seam streams the structured + // JSONStringsEachRowWithProgress format, so an explicit `FORMAT` clause would + // silently corrupt the tile (an empty successful-looking result, or ignored + // lines). Reject it with a clear error rather than mis-parse. + if (detectSqlFormat(execSql)) { + applyTileResult(app, q, slot, { + error: 'Dashboard panels require structured streaming results. Remove the explicit FORMAT clause.', + }); + onSettled(); + return; + } + const label = setSlotLoading(slot); + const ac = new AbortController(); + slot.abortController = ac; + const startedAt = app.now(); + // Client row limit = CAP (newResult trims + flags `capped`); server cap = + // CAP + 1 (the sentinel one past the client limit), so an exactly-CAP result + // is NOT marked truncated and a >CAP result is trimmed AND flagged (#193 req 1). + const result = newResult('Table', DASH_TILE_ROW_CAP); + await app.runReadInto(result, { + sql: execSql, + format: 'Table', + rowLimit: DASH_TILE_ROW_CAP + 1, + // readonly:2 rejects writes server-side (a favorite containing an INSERT/DDL + // is guarded, not executed); max_result_bytes bounds wide rows; param_ + // are the wave's prepared filter args (#173). + params: { readonly: 2, max_result_bytes: DASH_TILE_BYTE_CAP, ...mergedSourceArgs(src) }, + signal: ac.signal, + // Progress-only repaint (#193 design req 4): update the loading placeholder's + // row count as rows stream, never classify/render mid-stream. Updates the + // label captured for THIS request, so a superseded wave's late chunk can only + // touch its own (already-replaced) node. + onChunk: () => { label.textContent = 'Loading… ' + formatRows(result.progress.rows) + ' rows'; }, + }); + // Superseded mid-stream (a newer wave bumped the generation and aborted this + // request via supersedeSlot) or otherwise stale → discard silently: never + // render a partial/aborted result, never record recents. + if (slot.gen !== generation) return; + slot.abortController = null; + const r = dashboardTileResult(result, startedAt, app.now()); applyTileResult(app, q, slot, r); - // #171: this tile completed successfully — record its bound params (the - // exact wave's boundParams snapshot, so a param confined to an inactive - // optional block — never in `src.statements[*].boundParams` — is never - // recorded). A superseded/discarded fetch (the `return` above) never - // reaches here at all. + // #171: this tile completed (current generation) — record its bound params on + // success only (the exact wave's boundParams snapshot, so a param confined to + // an inactive optional block — never in `src.statements[*].boundParams` — is + // never recorded). An errored tile records nothing. if (r.error == null) app.recordBoundParams(src.statements.flatMap((s) => s.boundParams)); onSettled(); } @@ -392,20 +492,48 @@ export function renderDashboard(app) { } }; + // Build the wave's execution plan for a set of query-backed favorites: one + // `{ q, slot, src, generation }` per tile, reserving each slot's generation + // (and aborting any in-flight request) synchronously HERE, at wave creation + // (#193 design req 3). Reserving up front — not when a pool worker starts — + // closes the stale-wave race: a queued older worker sees `slot.gen !== + // generation` and discards itself instead of superseding a newer wave. + const planWave = (indices, wave) => indices + .filter((i) => !isTextFav(favorites[i])) + .map((i) => ({ q: favorites[i], slot: slots[i], src: wave[i], generation: supersedeSlot(slots[i]) })); + + const runPlan = (plan) => { + // Mark every planned slot loading up front — before the 6-way pool starts — + // so tiles beyond TILE_CONCURRENCY's window don't linger on stale content + // while queued. Applies to BOTH full Refresh and targeted affected waves + // (#193); runSlotTile re-marks its own slot loading when its worker starts + // (capturing the progress label), so filled tiles simply repaint identically. + plan.forEach(({ slot }) => setSlotLoading(slot)); + return runPool(plan, TILE_CONCURRENCY, + ({ q, slot, src, generation }) => runSlotTile(app, q, slot, updateSkipNote, src, generation)); + }; + // Re-run only the favorites whose SQL references `name` (a filter field's // debounced/committed edit, #149 D3) — not the whole grid. Affected-source // detection comes from the analysis (#173): `optionalIn` keeps a tile // affected even while the param's optional blocks are inactive (#165), so an // activation flip re-runs it exactly like a value change. A no-op before // the first successful run (slots not built yet). - function runAffected(name) { - if (!slots.length) return; + async function runAffected(name) { + if (!slots.length) return undefined; + // Match full Refresh: ONE token preflight before the wave (#193 design + // req 2). `runReadInto` leaves token freshness to the caller, so without + // this each affected tile would independently race a rotating-token refresh + // through authedFetch; a failed preflight issues no requests and drives + // sign-out exactly once, exactly like Refresh. + if (!(await app.ensureFreshToken())) { app.chCtx.onSignedOut(); return undefined; } const f = analysis.fields[name]; // the filter bar only renders analyzed params const affected = new Set(f.requiredIn.concat(f.optionalIn)); const wave = prepareWave(); - const targets = favorites.map((q, i) => i) - .filter((i) => !isTextFav(favorites[i]) && affected.has(tileId(i))); - return Promise.all(targets.map((i) => runSlotTile(app, favorites[i], slots[i], updateSkipNote, wave[i]))); + const targets = favorites.map((q, i) => i).filter((i) => affected.has(tileId(i))); + // Same 6-way pool as full Refresh (#193 design req 7): a wide filter change + // is bounded to TILE_CONCURRENCY concurrent reads, not an unbounded fan-out. + return runPlan(planWave(targets, wave)); } const runAll = async () => { @@ -422,19 +550,16 @@ export function renderDashboard(app) { // synchronously, before any tile query is issued — and they never join // the wave below (zero queries for a text favorite). slots.forEach((s, i) => { if (isTextFav(favorites[i])) renderTextSlot(app, favorites[i], s); }); - // Every query-backed favorite re-runs on a full refresh (unlike a filter's - // targeted runAffected). Mark every such slot loading up front rather than - // leaving tiles beyond TILE_CONCURRENCY's window showing stale content (or, - // on first load, an empty card) until the pool gets around to them. - slots.forEach((s, i) => { if (!isTextFav(favorites[i])) setSlotLoading(s); }); - // One prepared batch (and one wall-clock read) for the whole refresh wave. - const wave = prepareWave(); + // One prepared batch (and one wall-clock read) for the whole refresh wave; + // reserve every query-backed slot's generation NOW (planWave), before the + // pool starts, so a queued worker from an older Refresh discards itself. + // runPlan marks every planned slot loading up front (queued tiles included). + const plan = planWave(favorites.map((q, i) => i), prepareWave()); // try/finally so the button always re-enables and the timestamp always // updates — even if a tile render unexpectedly throws (runSlotTile itself // is total, so this is belt-and-suspenders against the pool rejecting). try { - await runPool(favorites, TILE_CONCURRENCY, - (q, i) => (isTextFav(q) ? undefined : runSlotTile(app, q, slots[i], updateSkipNote, wave[i]))); + await runPlan(plan); } finally { updated.textContent = 'Updated ' + new Date().toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }); refreshBtn.disabled = false; diff --git a/tests/unit/ch-client.test.js b/tests/unit/ch-client.test.js index 494c365c..b27bc24a 100644 --- a/tests/unit/ch-client.test.js +++ b/tests/unit/ch-client.test.js @@ -1,6 +1,6 @@ import { describe, it, expect, vi } from 'vitest'; import { - chUrl, authedFetch, queryJson, queryDashboardTile, loadServerVersion, loadSchema, loadColumns, loadReferenceData, loadEntityDoc, runQuery, killQuery, exportQuery, loadSchemaLineage, loadSchemaCards, loadLineageTransitive, loadTableDetail, AST_PROGRESSIVE_THRESHOLD, byUnderscoreThenName, + chUrl, authedFetch, queryJson, loadServerVersion, loadSchema, loadColumns, loadReferenceData, loadEntityDoc, runQuery, killQuery, exportQuery, loadSchemaLineage, loadSchemaCards, loadLineageTransitive, loadTableDetail, AST_PROGRESSIVE_THRESHOLD, byUnderscoreThenName, } from '../../src/net/ch-client.js'; import { sqlString } from '../../src/core/format.js'; @@ -56,41 +56,11 @@ describe('chUrl', () => { }); }); -describe('queryDashboardTile', () => { - it('runs read-only (readonly=2) + FORMAT JSON and returns parsed JSON', async () => { - const ctx = ctxWith(async () => jsonResp({ meta: [{ name: 'n', type: 'UInt64' }], data: [{ n: 1 }] })); - const out = await queryDashboardTile(ctx, 'SELECT 1 AS n\nFORMAT JSON'); - expect(out.data).toEqual([{ n: 1 }]); - const url = ctx.fetch.mock.calls[0][0]; - expect(url).toContain('default_format=JSON'); - expect(url).toContain('readonly=2'); - }); - it('requests the best-effort result caps (#149 D9): row cap + sentinel, byte cap, break overflow', async () => { - const ctx = ctxWith(async () => jsonResp({ meta: [], data: [] })); - await queryDashboardTile(ctx, 'SELECT 1\nFORMAT JSON'); - const url = ctx.fetch.mock.calls[0][0]; - expect(url).toContain('max_result_rows=5001'); // DASH_TILE_ROW_CAP + 1 — the client-trim truncation sentinel - expect(url).toContain('max_result_bytes=50000000'); - expect(url).toContain('result_overflow_mode=break'); - expect(url).toContain('readonly=2'); - }); - it('throws CH reason on a non-ok response', async () => { - const ctx = ctxWith(async () => textResp('Code: 164. DB::Exception: Cannot execute query in readonly mode', false, 500)); - await expect(queryDashboardTile(ctx, 'DROP TABLE t')).rejects.toThrow(/readonly mode/); - }); - it('forwards params as param_ query-string args (#149 D3)', async () => { - const ctx = ctxWith(async () => jsonResp({ meta: [], data: [] })); - await queryDashboardTile(ctx, 'SELECT {year:UInt16}\nFORMAT JSON', undefined, { param_year: '2024' }); - const url = ctx.fetch.mock.calls[0][0]; - expect(url).toContain('param_year=2024'); - }); - it('omits params entirely when not passed (backward compatible)', async () => { - const ctx = ctxWith(async () => jsonResp({ meta: [], data: [] })); - await queryDashboardTile(ctx, 'SELECT 1\nFORMAT JSON'); - const url = ctx.fetch.mock.calls[0][0]; - expect(url).not.toContain('param_'); - }); -}); +// (queryDashboardTile was retired in #193 — dashboard tiles now stream through +// runQuery via the shared app.runReadInto seam, carrying readonly:2 / +// max_result_bytes / param_* in `params` and capping with resultRowLimit. Its +// URL-shaping is covered by runQuery's tests below; the dashboard's use of the +// seam is covered in dashboard.test.js.) describe('authedFetch', () => { it('throws + signals out when no token', async () => { diff --git a/tests/unit/dashboard.test.js b/tests/unit/dashboard.test.js index b1d39f13..35827197 100644 --- a/tests/unit/dashboard.test.js +++ b/tests/unit/dashboard.test.js @@ -1,8 +1,8 @@ import { describe, it, expect, vi } from 'vitest'; import { webcrypto } from 'node:crypto'; import { - isDashboardRoute, configBase, dashboardTileSql, parseJsonResult, - normalizeDashLayout, normalizeDashCols, DASH_TILE_ROW_CAP, DASH_TABLE_DISPLAY_CAP, + isDashboardRoute, configBase, + normalizeDashLayout, normalizeDashCols, DASH_TILE_ROW_CAP, DASH_TILE_BYTE_CAP, DASH_TABLE_DISPLAY_CAP, activeDashboardView, dashboardViewSelection, } from '../../src/core/dashboard.js'; import { CHART_ROW_CAPS } from '../../src/core/chart-data.js'; @@ -11,6 +11,7 @@ import { snapshotAuth, restoreAuth, hasAuth, isAuthRequest, isAuthGrant, } from '../../src/core/auth-handoff.js'; import { renderDashboard } from '../../src/ui/dashboard.js'; +import { applyStreamLine } from '../../src/core/stream.js'; import { emptyRecentMap, recordRecent } from '../../src/core/recent-values.js'; import { makeApp, FakeChart } from '../helpers/fake-app.js'; import { createApp } from '../../src/ui/app.js'; @@ -37,71 +38,10 @@ describe('configBase', () => { }); }); -describe('dashboardTileSql', () => { - it('strips a trailing ; and appends FORMAT JSON', () => { - expect(dashboardTileSql('SELECT 1;')).toBe('SELECT 1\nFORMAT JSON'); - expect(dashboardTileSql('SELECT 1')).toBe('SELECT 1\nFORMAT JSON'); - }); - it('leaves an explicit FORMAT clause intact (no double FORMAT)', () => { - expect(dashboardTileSql('SELECT 1 FORMAT CSV')).toBe('SELECT 1 FORMAT CSV'); - expect(dashboardTileSql('SELECT 1 FORMAT JSON;')).toBe('SELECT 1 FORMAT JSON'); - // FORMAT followed by SETTINGS (either-order clause) must still count as trailing. - expect(dashboardTileSql('SELECT 1 FORMAT JSON SETTINGS max_threads=1')) - .toBe('SELECT 1 FORMAT JSON SETTINGS max_threads=1'); - }); - it('peels a trailing comment so an existing FORMAT is not doubled', () => { - expect(dashboardTileSql('SELECT 1 FORMAT JSON -- daily')).toBe('SELECT 1 FORMAT JSON'); - expect(dashboardTileSql('SELECT 1 /* note */')).toBe('SELECT 1\nFORMAT JSON'); - }); - it('is defensive about empty/absent SQL (empty in → empty out)', () => { - expect(dashboardTileSql('')).toBe(''); - expect(dashboardTileSql(undefined)).toBe(''); - }); -}); - -describe('parseJsonResult', () => { - const nData = (n) => Array.from({ length: n }, (_, i) => ({ n: i })); - it('transforms a full FORMAT JSON response into columns + array rows + meta', () => { - const out = parseJsonResult({ - meta: [{ name: 'k', type: 'String' }, { name: 'v', type: 'UInt64' }], - data: [{ k: 'a', v: 1 }, { k: 'b', v: 2 }], - rows: 2, - statistics: { elapsed: 0.012, bytes_read: 2048 }, - }); - expect(out.columns.map((c) => c.name)).toEqual(['k', 'v']); - expect(out.rows).toEqual([['a', 1], ['b', 2]]); - expect(out.meta).toEqual({ rows: 2, ms: 12, bytes: 2048, truncated: false }); - }); - it('is defensive about a bare response (no meta/data/statistics/rows)', () => { - const out = parseJsonResult({}); - expect(out.columns).toEqual([]); - expect(out.rows).toEqual([]); - expect(out.meta).toEqual({ rows: 0, ms: null, bytes: null, truncated: false }); - }); - it('exactly cap rows → kept whole, not truncated (#149 D9)', () => { - const out = parseJsonResult({ meta: [{ name: 'n', type: 'UInt64' }], data: nData(3) }, 3); - expect(out.rows).toEqual([[0], [1], [2]]); - expect(out.meta.rows).toBe(3); - expect(out.meta.truncated).toBe(false); - }); - it('cap+1 rows → sliced to cap, truncated, and meta.rows is rows SHOWN (json.rows ignored)', () => { - const out = parseJsonResult( - // json.rows is a block-boundary overshoot count under result_overflow_mode - // 'break' — neither the full nor the displayed count, so never surfaced. - { meta: [{ name: 'n', type: 'UInt64' }], data: nData(4), rows: 9999 }, - 3, - ); - expect(out.rows).toEqual([[0], [1], [2]]); - expect(out.meta.rows).toBe(3); - expect(out.meta.truncated).toBe(true); - }); - it('an uncapped call reports rows shown with truncated false, even when json.rows disagrees', () => { - const out = parseJsonResult({ meta: [{ name: 'n', type: 'UInt64' }], data: nData(2), rows: 9999 }); - expect(out.rows).toEqual([[0], [1]]); - expect(out.meta.rows).toBe(2); - expect(out.meta.truncated).toBe(false); - }); -}); +// (dashboardTileSql + parseJsonResult were retired in #193 — the tiles stream +// through the shared app.runReadInto seam, so SQL prep is now just the shared +// materialization (#165) and the client row bound is newResult's trim + `capped` +// flag. The tile↔seam wiring is covered under `renderDashboard` below.) describe('DASH_TILE_ROW_CAP', () => { // The invariant the constant's docstring states, enforced: a fetch cap below @@ -213,8 +153,37 @@ const chartResult = (meta = { rows: 2, ms: 5, bytes: 100 }) => ({ }); const kpiResult = () => ({ columns: [{ name: 'value', type: 'UInt64' }], rows: [[42]], meta: { rows: 1, ms: 1, bytes: 10 } }); +// Bridge the legacy tile-outcome fixtures onto the streaming `app.runReadInto` +// seam (#193): `spy(sql, param_* args)` returns the logical tile outcome +// ({columns, rows, meta} | {error, cancelled}); `streamInto` folds it into the +// caller-owned result exactly as ClickHouse's JSONStrings…Progress stream would +// (columns, rows, progress bytes, and the `capped` flag from meta.truncated), +// then fires onChunk once. Keeping the spy's (sql, params) signature lets the +// existing call-count/arg assertions ride unchanged — only the param_* subset +// reaches the spy (the seam's readonly:2 / max_result_bytes / rowLimit are +// asserted separately, on the runReadInto opts). Returns the runReadInto mock. +function streamInto(spy) { + return vi.fn(async (result, opts = {}) => { + const params = opts.params || {}; + const paramArgs = Object.fromEntries(Object.entries(params).filter(([k]) => k.startsWith('param_'))); + const out = await spy(opts.sql, paramArgs); + if (out.error != null) { result.error = out.error; return result; } + if (out.cancelled) { result.cancelled = true; return result; } + result.columns = out.columns || []; + result.rows = (out.rows || []).slice(); + result.progress = { ...result.progress, rows: result.rows.length, bytes: (out.meta && out.meta.bytes) || 0 }; + result.capped = !!(out.meta && out.meta.truncated); + if (opts.onChunk) opts.onChunk(); + return result; + }); +} + +// Build a dashboard app whose tiles run through the seam via `streamInto`. The +// `runTile` spy is exposed as `app.tileSpy` for the few call-count assertions +// that referenced the old `app.runTile`. function dashApp(favorites, runTile) { - const app = makeApp({ runTile }); + const app = makeApp({ runReadInto: streamInto(runTile) }); + app.tileSpy = runTile; app.state.savedQueries = favorites; return app; } @@ -279,11 +248,17 @@ describe('renderDashboard', () => { expect(app.root.querySelector('.dash-skip').style.display).toBe('none'); // an error is not a skip }); - it('omits ms/bytes from the footer when CH did not report them', async () => { + it('the footer always shows rows · ms · bytes on the streaming seam (#193: wall-clock ms + progress bytes)', async () => { + // Unlike the old FORMAT-JSON path (which omitted stats CH did not report), + // the streaming seam always has a wall-clock ms and a progress byte count + // (0 when none streamed), so the footer row is unconditional — three spans. const app = dashApp([{ id: '1', name: 'Q', sql: 'q', favorite: true }], - vi.fn(async () => chartResult({ rows: 2, ms: null, bytes: null }))); + vi.fn(async () => chartResult({ rows: 2, bytes: 0 }))); await renderDashboard(app); - expect(app.root.querySelector('.dash-tile-foot').children.length).toBe(1); + const foot = app.root.querySelector('.dash-tile-foot'); + expect(foot.children.length).toBe(3); + expect(foot.textContent).toContain('0 ms'); + expect(foot.textContent).toContain('scanned'); }); it('a fetch-truncated tile gets the honest "first N rows fetched" footer note (#149 D9)', async () => { @@ -296,7 +271,7 @@ describe('renderDashboard', () => { it('has a theme toggle wired to app.toggleTheme', async () => { const toggleTheme = vi.fn(); - const app = makeApp({ runTile: vi.fn(async () => chartResult()), toggleTheme }); + const app = makeApp({ runReadInto: streamInto(vi.fn(async () => chartResult())), toggleTheme }); app.state.theme = 'dark'; // exercise the dark-theme icon branch app.state.savedQueries = [{ id: '1', name: 'Q', sql: 'q', favorite: true }]; await renderDashboard(app); @@ -309,7 +284,7 @@ describe('renderDashboard', () => { it('redirects to login once (no tiles) when the session cannot be refreshed', async () => { const onSignedOut = vi.fn(); const app = makeApp({ - runTile: vi.fn(async () => chartResult()), + runReadInto: streamInto(vi.fn(async () => chartResult())), ensureFreshToken: vi.fn(async () => false), chCtx: { onSignedOut }, }); @@ -319,7 +294,7 @@ describe('renderDashboard', () => { ]; await renderDashboard(app); expect(onSignedOut).toHaveBeenCalledTimes(1); // one redirect, not one per tile - expect(app.runTile).not.toHaveBeenCalled(); + expect(app.runReadInto).not.toHaveBeenCalled(); expect(app.root.querySelectorAll('.dash-tile').length).toBe(0); }); @@ -518,11 +493,225 @@ describe('renderDashboard', () => { it('changing layout never re-runs tile queries', async () => { const app = oneFav(); await renderDashboard(app); - expect(app.runTile).toHaveBeenCalledTimes(1); // the initial render + expect(app.runReadInto).toHaveBeenCalledTimes(1); // the initial render seg(app.root, 'Full width').dispatchEvent(new Event('click', { bubbles: true })); seg(app.root, 'Report').dispatchEvent(new Event('click', { bubbles: true })); seg(app.root, '2 columns').dispatchEvent(new Event('click', { bubbles: true })); - expect(app.runTile).toHaveBeenCalledTimes(1); // presentation-only — no refetch + expect(app.runReadInto).toHaveBeenCalledTimes(1); // presentation-only — no refetch + }); +}); + +// ── #193: tiles on the shared streaming app.runReadInto seam ───────────────── +describe('renderDashboard — streaming seam (#193)', () => { + const flush = () => new Promise((resolve) => setTimeout(resolve, 0)); + const yearInput = (root) => root.querySelector('.var-field input[aria-label="year"]'); + const commit = (input, value) => { + input.value = value; + input.dispatchEvent(new Event('input', { bubbles: true })); + input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })); + }; + const paramFav = (id, table = id) => ({ id, name: id, sql: `SELECT * FROM ${table} WHERE y = {year:UInt16}`, favorite: true }); + + it('streams read-only with the row-cap split, readonly/byte caps, param args, and a signal (req 1/3)', async () => { + const app = dashApp([{ id: '1', name: 'Q', sql: 'SELECT {year:UInt16} AS n', favorite: true }], + vi.fn(async () => chartResult())); + app.state.varValues = { year: '2024' }; + await renderDashboard(app); + expect(app.runReadInto).toHaveBeenCalledTimes(1); + const [result, opts] = app.runReadInto.mock.calls[0]; + expect(opts.format).toBe('Table'); + expect(opts.rowLimit).toBe(DASH_TILE_ROW_CAP + 1); // server max_result_rows = CAP + 1 (sentinel) + expect(result.rowLimit).toBe(DASH_TILE_ROW_CAP); // client-side trim = CAP + expect(opts.params).toMatchObject({ readonly: 2, max_result_bytes: DASH_TILE_BYTE_CAP, param_year: '2024' }); + expect(opts.signal).toBeTruthy(); // an AbortController signal → real per-tile cancellation + }); + + it('exactly-CAP is not truncated; CAP+1 is trimmed AND flagged (req 1, via the real applyStreamLine)', async () => { + // Stream N single-column rows through the REAL accumulator so the client cap + // (newResult('Table', CAP)) trims + flags exactly as production would. + const streamN = (n) => vi.fn(async (result, opts) => { + applyStreamLine({ meta: [{ name: 'n', type: 'UInt64' }] }, result); + for (let i = 0; i < n; i++) applyStreamLine({ row: { n: i } }, result); + applyStreamLine({ progress: { read_rows: n, read_bytes: 10 } }, result); + opts.onChunk(); + return result; + }); + const fav = [{ id: '1', name: 'Q', sql: 'q', favorite: true, panel: { cfg: { type: 'table' } } }]; + + const exact = makeApp({ runReadInto: streamN(DASH_TILE_ROW_CAP) }); + exact.state.savedQueries = fav; + await renderDashboard(exact); + expect(exact.root.querySelector('.dash-tile-foot').textContent).not.toContain('rows fetched'); + + const over = makeApp({ runReadInto: streamN(DASH_TILE_ROW_CAP + 1) }); + over.state.savedQueries = fav; + await renderDashboard(over); + const foot = over.root.querySelector('.dash-tile-foot').textContent; + expect(foot).toContain('first ' + DASH_TILE_ROW_CAP.toLocaleString() + ' rows fetched'); + expect(foot).toContain(DASH_TILE_ROW_CAP.toLocaleString() + ' rows'); // rows SHOWN = trimmed CAP, not CAP+1 + }); + + it('updates only the loading placeholder as rows stream — never classifies mid-stream (req 4)', async () => { + const runReadInto = vi.fn((result, opts) => { + applyStreamLine({ meta: [{ name: 'k', type: 'String' }, { name: 'v', type: 'UInt64' }] }, result); + applyStreamLine({ row: { k: 'a', v: '1' } }, result); + applyStreamLine({ progress: { read_rows: 1420, read_bytes: 10 } }, result); + opts.onChunk(); // mid-stream repaint + return new Promise(() => {}); // never settles — stay in the loading state + }); + const app = makeApp({ runReadInto }); + app.state.savedQueries = [{ id: '1', name: 'Q', sql: 'q', favorite: true }]; + renderDashboard(app); + await flush(); + const load = app.root.querySelector('.dash-tile-load'); + expect(load).not.toBeNull(); + expect(load.textContent).toBe('Loading… 1.4K rows'); // progress row count (formatRows compact), placeholder only + expect(app.root.querySelector('.dash-tile canvas')).toBeNull(); // NOT classified/charted yet + expect(app.root.querySelector('.dash-tile-foot').textContent).toBe(''); // no footer mid-stream + }); + + it('rejects an explicit FORMAT clause with a clear error and issues no request (req 5)', async () => { + const spy = vi.fn(async () => chartResult()); + const app = dashApp([{ id: '1', name: 'Q', sql: 'SELECT 1 FORMAT JSON', favorite: true }], spy); + await renderDashboard(app); + expect(app.root.querySelector('.dash-tile-error').textContent) + .toContain('Remove the explicit FORMAT clause'); + expect(app.runReadInto).not.toHaveBeenCalled(); // never mis-parsed as a structured stream + }); + + it('full Refresh performs exactly one token preflight before fanning out (req 2)', async () => { + const app = dashApp([ + { id: '1', name: 'A', sql: 'a', favorite: true }, + { id: '2', name: 'B', sql: 'b', favorite: true }, + ], vi.fn(async () => chartResult())); + await renderDashboard(app); + expect(app.ensureFreshToken).toHaveBeenCalledTimes(1); // once for the whole wave, not per tile + }); + + it('an affected-filter wave preflights once; a failed preflight issues no requests and signs out (req 2)', async () => { + const app = dashApp([paramFav('1', 't')], vi.fn(async () => chartResult())); + app.state.varValues = { year: '1' }; + await renderDashboard(app); + expect(app.ensureFreshToken).toHaveBeenCalledTimes(1); // the initial refresh + app.runReadInto.mockClear(); + app.ensureFreshToken.mockResolvedValue(false); // session lost before the affected wave + commit(yearInput(app.root), '2'); + await flush(); + expect(app.ensureFreshToken).toHaveBeenCalledTimes(2); // one preflight for the affected wave + expect(app.chCtx.onSignedOut).toHaveBeenCalledTimes(1); + expect(app.runReadInto).not.toHaveBeenCalled(); // failed preflight → no tile requests + }); + + it('a newer wave aborts the previous slot request at wave creation (generation reserved up front, req 3/5)', async () => { + const signals = []; + const resolvers = []; + const runReadInto = vi.fn((result, opts) => { + signals.push(opts.signal); + return new Promise((res) => resolvers.push(() => { result.columns = [{ name: 'k', type: 'String' }]; result.rows = [['a']]; res(result); })); + }); + const app = makeApp({ runReadInto }); + app.state.savedQueries = [paramFav('1', 't')]; + app.state.varValues = { year: '1' }; + const rendered = renderDashboard(app); + await flush(); + expect(signals).toHaveLength(1); + resolvers.splice(0).forEach((r) => r()); + await rendered; + const input = yearInput(app.root); + commit(input, '11'); // wave A + await flush(); + expect(signals).toHaveLength(2); + commit(input, '22'); // wave B — created before A's request settled + await flush(); + expect(signals).toHaveLength(3); + expect(signals[1].aborted).toBe(true); // A superseded at B's CREATION, before A resolved + expect(signals[2].aborted).toBe(false); + resolvers.splice(0).forEach((r) => r()); // drain (both A and B) — no throw + await flush(); + }); + + it('a queued Refresh worker superseded by a newer wave discards itself without issuing (req 3/5/7)', async () => { + const calls = []; + const resolvers = []; + const runReadInto = vi.fn((result, opts) => { + calls.push(opts.params.param_year); + return new Promise((res) => resolvers.push(() => { result.columns = [{ name: 'k', type: 'String' }]; result.rows = [['a']]; res(result); })); + }); + const app = makeApp({ runReadInto }); + app.state.savedQueries = Array.from({ length: 8 }, (_, i) => paramFav(String(i), 't' + i)); + app.state.varValues = { year: '1' }; + const rendered = renderDashboard(app); // wave A (full Refresh) + await flush(); + expect(calls.filter((v) => v === '1')).toHaveLength(6); // TILE_CONCURRENCY: 6 in flight, 2 queued + // Wave B (a filter change) supersedes every slot at CREATION — before A's + // queued workers ever reach tiles 6 & 7. + commit(yearInput(app.root), '2'); + await flush(); + expect(calls.filter((v) => v === '2')).toHaveLength(6); // B fans out its own 6 + // Drain everything; A's two queued workers dequeue AFTER B superseded them, + // see the stale generation, and discard WITHOUT issuing a year=1 request. + while (resolvers.length) { resolvers.splice(0).forEach((r) => r()); await flush(); } + await rendered; + expect(calls.filter((v) => v === '1')).toHaveLength(6); // A never issued the 2 queued + expect(calls.filter((v) => v === '2')).toHaveLength(8); // B issued all 8 + }); + + it('an affected wave is bounded to the same 6-way pool as full Refresh (req 7)', async () => { + const resolvers = []; + const runReadInto = vi.fn((result, opts) => new Promise((res) => resolvers.push(() => { result.columns = [{ name: 'k', type: 'String' }]; result.rows = [['a']]; res(result); }))); + const app = makeApp({ runReadInto }); + app.state.savedQueries = Array.from({ length: 8 }, (_, i) => paramFav(String(i), 't' + i)); + app.state.varValues = { year: '1' }; + const rendered = renderDashboard(app); + await flush(); + expect(resolvers).toHaveLength(6); // initial full refresh caps at 6 + while (resolvers.length) { resolvers.splice(0).forEach((r) => r()); await flush(); } + await rendered; + const before = runReadInto.mock.calls.length; // 8 + commit(yearInput(app.root), '2'); + await flush(); + expect(runReadInto.mock.calls.length - before).toBe(6); // the affected wave also caps at 6 concurrent + // …but every affected tile shows the loading placeholder up front (not just + // the 6 in flight) — no queued tile lingers on stale content while waiting. + expect(app.root.querySelectorAll('.dash-tile-load')).toHaveLength(8); + while (resolvers.length) { resolvers.splice(0).forEach((r) => r()); await flush(); } + }); + + it('a stale (superseded) response neither renders nor records recents (req 6)', async () => { + const resolvers = []; + const runReadInto = vi.fn((result, opts) => new Promise((res) => resolvers.push((out) => { + Object.assign(result, out); + res(result); + }))); + const app = makeApp({ runReadInto }); + app.state.savedQueries = [paramFav('1', 't')]; + app.state.varValues = { year: '1' }; + const rendered = renderDashboard(app); + await flush(); + // ≥2 rows so the tile renders a table (a 1-row unconfigured result is a KPI skip). + resolvers.splice(0).forEach((r) => r({ columns: [{ name: 'k', type: 'String' }], rows: [['a'], ['a2']] })); + await rendered; + app.recordBoundParams.mockClear(); + const input = yearInput(app.root); + commit(input, '11'); // wave A (superseded below) + await flush(); + commit(input, '22'); // wave B supersedes A + await flush(); + // B resolves first (current), then the stale A resolves late. + resolvers[1]({ columns: [{ name: 'k', type: 'String' }], rows: [['B'], ['B2']] }); + await flush(); + resolvers[0]({ columns: [{ name: 'k', type: 'String' }], rows: [['A-stale'], ['A2']] }); + await flush(); + expect(app.root.querySelector('.dash-tile').textContent).toContain('B'); // B rendered + expect(app.root.querySelector('.dash-tile').textContent).not.toContain('A-stale'); + expect(app.recordBoundParams).toHaveBeenCalledTimes(1); // only B recorded; the stale A did not + }); + + it('never touches workbench run state — tiles own their own results (req: isolation)', async () => { + const app = dashApp([{ id: '1', name: 'Q', sql: 'q', favorite: true }], vi.fn(async () => chartResult())); + await renderDashboard(app); + expect(app.state.running.value).toBe(false); // dashboard tiles never flip the workbench run signal + expect(app.activeTab().result).toBeFalsy(); // no active-tab result written }); }); @@ -924,7 +1113,7 @@ describe('renderDashboard — global filter bar (#149 D3)', () => { await rendered; // Distinct but still UInt16-valid values (#170: an invalid value would - // never reach app.runTile at all, short-circuiting this race). + // never reach the seam at all, short-circuiting this race). const input = fieldInput(app.root, 'year'); setInput(input, '11'); pressEnter(input); @@ -1162,7 +1351,7 @@ describe('renderDashboard — global filter bar (#149 D3)', () => { paramFav('2', 'SELECT * FROM u WHERE d >= {from:DateTime}'), ]; const runTile = vi.fn(async () => chartResult()); - const app = makeApp({ runTile, wallNow: vi.fn(() => 1751200000000) }); + const app = makeApp({ runReadInto: streamInto(runTile), wallNow: vi.fn(() => 1751200000000) }); app.state.savedQueries = favorites; app.state.varValues = { from: '-1h' }; await renderDashboard(app); @@ -1316,7 +1505,7 @@ describe('renderDashboard — recent values (#171)', () => { }); }); -// ── app.js: runTile + auth handoff wiring ──────────────────────────────────── +// ── app.js: dashboard render + auth handoff wiring ─────────────────────────── function jwt(payload) { const b = (o) => Buffer.from(JSON.stringify(o)).toString('base64url'); return `${b({ alg: 'RS256' })}.${b(payload)}.sig`; @@ -1328,9 +1517,21 @@ function resp(opts) { ok: opts.ok ?? true, status: opts.status ?? 200, json: async () => opts.json, text: async () => opts.text ?? JSON.stringify(opts.json), clone() { return this; }, + body: opts.body, headers: { get: () => null }, }; } +// A streaming response body (JSONStringsEachRowWithProgress lines), for the +// tile/run() path that reads resp.body.getReader() rather than resp.json(). +function streamBody(lines) { + let i = 0; + return { + getReader: () => ({ + read: async () => (i < lines.length ? { done: false, value: new TextEncoder().encode(lines[i++]) } : { done: true }), + releaseLock: () => {}, + }), + }; +} function makeFetch(routes) { return vi.fn(async (url, init) => { const sql = init && init.body; @@ -1357,73 +1558,6 @@ const msg = (data, source, origin = 'https://ch.example') => { return e; }; -describe('app.runTile', () => { - it('returns the parsed result on success', async () => { - const app = createApp(appEnv({ - fetch: makeFetch([[(u, sql) => /SELECT k/.test(sql || ''), - resp({ json: { meta: [{ name: 'k', type: 'String' }, { name: 'v', type: 'UInt64' }], data: [{ k: 'a', v: 1 }], statistics: { elapsed: 0.01, bytes_read: 2048 } } })]]), - })); - const r = await app.runTile('SELECT k, v FROM t'); - expect(r.columns.map((c) => c.name)).toEqual(['k', 'v']); - expect(r.rows).toEqual([['a', 1]]); - expect(r.meta).toMatchObject({ ms: 10, bytes: 2048 }); - }); - it('reports the CH error message on a rejected query', async () => { - const app = createApp(appEnv({ - fetch: makeFetch([[(u, sql) => /SELECT/.test(sql || ''), resp({ ok: false, status: 500, text: 'Cannot execute query in readonly mode' })]]), - })); - expect((await app.runTile('SELECT 1')).error).toMatch(/readonly/); - }); - it('substitutes state.varValues as param_ args (#149 D3)', async () => { - const fetch = makeFetch([[(u, sql) => /SELECT/.test(sql || ''), - resp({ json: { meta: [{ name: 'n', type: 'UInt64' }], data: [{ n: 1 }] } })]]); - const app = createApp(appEnv({ fetch })); - app.state.varValues.year = '2024'; - await app.runTile('SELECT {year:UInt16} AS n'); - const queryCall = fetch.mock.calls.find((c) => c[1] && c[1].method === 'POST'); - expect(queryCall[0]).toContain('param_year=2024'); - }); - it('prepares per-statement, so a multi-statement favorite still binds its params (#155)', async () => { - // paramArgs over the whole blob saw the leading SET and skipped substitution - // entirely; the pipeline splits first, so param_year rides along. - const fetch = makeFetch([[(u, sql) => /SELECT/.test(sql || ''), - resp({ json: { meta: [{ name: 'n', type: 'UInt64' }], data: [{ n: 1 }] } })]]); - const app = createApp(appEnv({ fetch })); - app.state.varValues.year = '2024'; - await app.runTile('SET x = 1; SELECT {year:UInt16} AS n'); - const queryCall = fetch.mock.calls.find((c) => c[1] && c[1].method === 'POST'); - expect(queryCall[0]).toContain('param_year=2024'); - }); - it('explicit prepared args win over self-preparation (the dashboard wave passes them)', async () => { - const fetch = makeFetch([[(u, sql) => /SELECT/.test(sql || ''), - resp({ json: { meta: [{ name: 'n', type: 'UInt64' }], data: [{ n: 1 }] } })]]); - const app = createApp(appEnv({ fetch })); - app.state.varValues.year = '1999'; // must NOT be read — the caller's batch is authoritative - await app.runTile('SELECT {year:UInt16} AS n', { param_year: '2024' }); - const queryCall = fetch.mock.calls.find((c) => c[1] && c[1].method === 'POST'); - expect(queryCall[0]).toContain('param_year=2024'); - expect(queryCall[0]).not.toContain('param_year=1999'); - }); - it('errors (without driving sign-out) when there is no token', async () => { - const app = createApp(appEnv({ sessionStorage: memSession({}) })); - expect(await app.runTile('SELECT 1')).toEqual({ error: 'Not signed in' }); - }); - it('trims an over-cap result to DASH_TILE_ROW_CAP and flags meta.truncated (#149 D9)', async () => { - // Server sentinel (max_result_rows = cap + 1, overflow 'break') delivered - // one row past the cap — the client trim is the guaranteed bound. - const data = Array.from({ length: DASH_TILE_ROW_CAP + 1 }, (_, i) => ({ n: i })); - const app = createApp(appEnv({ - fetch: makeFetch([[(u, sql) => /SELECT n/.test(sql || ''), - resp({ json: { meta: [{ name: 'n', type: 'UInt64' }], data, rows: data.length } })]]), - })); - const r = await app.runTile('SELECT n FROM big'); - expect(r.rows).toHaveLength(DASH_TILE_ROW_CAP); - expect(r.rows[DASH_TILE_ROW_CAP - 1]).toEqual([DASH_TILE_ROW_CAP - 1]); - expect(r.meta.rows).toBe(DASH_TILE_ROW_CAP); - expect(r.meta.truncated).toBe(true); - }); -}); - describe('app config base on the dashboard route', () => { it('resolves config.json from /sql, not /sql/dashboard', async () => { const fetch = makeFetch([]); @@ -1439,14 +1573,24 @@ describe('app config base on the dashboard route', () => { }); describe('app.renderDashboard', () => { - it('renders the favorites dashboard into the root', async () => { - const app = createApp(appEnv({ - fetch: makeFetch([[(u, sql) => /mychart/.test(sql || ''), - resp({ json: { meta: [{ name: 'k', type: 'String' }, { name: 'v', type: 'UInt64' }], data: [{ k: 'a', v: 1 }, { k: 'b', v: 2 }] } })]]), - })); + it('renders the favorites dashboard into the root — streaming the tile through the real seam (#193)', async () => { + // End-to-end through createApp's real app.runReadInto → ch.runQuery → the + // streaming JSONStringsEachRowWithProgress reader (not resp.json()), the + // same transport run() and the detached view use. + const fetch = makeFetch([[(u, sql) => /mychart/.test(sql || ''), resp({ + body: streamBody([ + '{"meta":[{"name":"k","type":"String"},{"name":"v","type":"UInt64"}]}\n', + '{"row":{"k":"a","v":"1"}}\n', + '{"row":{"k":"b","v":"2"}}\n', + ]), + })]]); + const app = createApp(appEnv({ fetch })); app.state.savedQueries = [{ id: '1', name: 'Q', sql: 'SELECT k, v FROM mychart', favorite: true }]; await app.renderDashboard(); expect(app.root.querySelector('.dash-tile canvas')).not.toBeNull(); + // The read-only tile guard (readonly=2) + the row-cap sentinel reach the wire. + expect(fetch.mock.calls.some((c) => /readonly=2/.test(c[0]))).toBe(true); + expect(fetch.mock.calls.some((c) => /max_result_rows=5001/.test(c[0]))).toBe(true); }); });