diff --git a/Darling/Darling.Tests/CollectorRunnerConnectionEngineTests.cs b/Darling/Darling.Tests/CollectorRunnerConnectionEngineTests.cs index dcbf4edcb0..3857258d41 100644 --- a/Darling/Darling.Tests/CollectorRunnerConnectionEngineTests.cs +++ b/Darling/Darling.Tests/CollectorRunnerConnectionEngineTests.cs @@ -27,8 +27,8 @@ namespace Darling.Tests; /// /// What this exists to catch. The runner had two paths: the per-database branch resolved /// TargetProviders.For(target) correctly, and the branch serving everything else did -/// new SqlConnection(server.ConnectionString) literally. Six of the seven PostgreSQL collectors take -/// the second path — only pg_autovacuum_stats fans out — so they were handed a SQL Server connection. +/// new SqlConnection(server.ConnectionString) literally. Six of the seven PostgreSQL collectors then +/// shipping took the second path — only pg_autovacuum_stats fanned out — so they were handed a SQL Server connection. /// SqlClient rejects Npgsql's keywords while PARSING the connection string, before any query runs /// ("Keyword not supported: 'host'"), and the resulting ArgumentException is neither /// SqlException nor PostgresException, so it missed both fault-classification arms and recorded diff --git a/Darling/Darling.Tests/ReadmeDerivedCountPinTests.cs b/Darling/Darling.Tests/ReadmeDerivedCountPinTests.cs index 775129c019..f355f6ab5c 100644 --- a/Darling/Darling.Tests/ReadmeDerivedCountPinTests.cs +++ b/Darling/Darling.Tests/ReadmeDerivedCountPinTests.cs @@ -292,6 +292,78 @@ public void TestConnectionTranscript_DenominatorsMatchThePostgresCollectorCount( Assert.All(denominators, d => Assert.Equal(PostgresCollectors, d)); } + /// + /// The "What gets collected" table names EVERY PostgreSQL collector, in both directions (#3261). It sat + /// at 12 of 27 across five collector waves with every check green, because nothing failed when a + /// collector shipped without a row — the exact defect this file exists for, one resolution finer than + /// a count. + /// + /// A bespoke fact rather than a entry, and the reason is the shared + /// mutation proof: drops the FIRST occurrence of each + /// member's backticked name inside the span, and several collector names legitimately appear inside + /// OTHER rows before their own — pg_replication_slots is a SOURCE of pg_xmin_horizon, + /// whose row sits above its own. Dropping that occurrence would leave extraction intact and the shared + /// proof would report this pin as vacuous when it is not. So this fact carries its own drift proof, + /// one that deletes a whole ROW instead of a token. + /// + [Fact] + public void WhatGetsCollectedTable_NamesEveryPostgresCollector() + { + var readme = ReadReadme(); + var span = WhatGetsCollectedSpan(readme); + + var expected = CollectorCatalog.All + .Where(c => c.TargetEngine == CollectorTargetEngine.PostgreSql) + .Select(c => c.Name) + .ToHashSet(StringComparer.Ordinal); + var actual = RowLeadingNames(span); + + var missing = expected.Except(actual, StringComparer.Ordinal) + .OrderBy(v => v, StringComparer.Ordinal).ToArray(); + var extra = actual.Except(expected, StringComparer.Ordinal) + .OrderBy(v => v, StringComparer.Ordinal).ToArray(); + + Assert.True(missing.Length == 0, + "the What-gets-collected table: the collector catalog ships " + string.Join(", ", missing) + + ", which Darling/README.md's table has no row for. Add the row — a collector the inventory " + + "documentation does not name is one nobody installing this product knows they are running."); + + Assert.True(extra.Length == 0, + "the What-gets-collected table: Darling/README.md has a row for " + string.Join(", ", extra) + + ", which is not a PostgreSQL collector in the catalog. Remove the row or fix its name."); + + /* The drift proof: deleting one collector's whole row must surface as a missing member. The victim + is a name that appears in no other row's cells, so the proof stays about the ROW being read. */ + var victimLine = span.Split('\n') + .First(l => l.StartsWith("| `pg_wraparound_stats`", StringComparison.Ordinal)); + var mutated = span.Replace(victimLine, string.Empty, StringComparison.Ordinal); + Assert.NotEqual(span, mutated); + Assert.DoesNotContain("pg_wraparound_stats", RowLeadingNames(mutated)); + } + + /// The table's rows: everything between the header row and the paragraph after the table. + private static string WhatGetsCollectedSpan(string readme) + { + var match = Regex.Match( + readme, + @"\| Collector \| Source \| Cadence / retention \| Why it exists \|(.+?)\r?\n\r?\nThe two Aurora-only collectors", + RegexOptions.Singleline); + + Assert.True(match.Success, + "Darling/README.md no longer contains the What-gets-collected table in the pinned shape " + + "(its header row, then rows, then a blank line, then the 'The two Aurora-only collectors' " + + "paragraph). It restates the catalog's PostgreSQL roster, so keep it parseable — a reworded " + + "header or a moved table has to fail here rather than quietly stop being checked."); + + return match.Groups[1].Value; + } + + /// Each row's FIRST cell — the collector the row is about, never a name inside its prose. + private static IReadOnlySet RowLeadingNames(string span) => + Regex.Matches(span, @"(?m)^\| `([a-z0-9_]+)`") + .Select(m => m.Groups[1].Value) + .ToHashSet(StringComparer.Ordinal); + /// /// Non-vacuity, the way /// does it: mutate a COPY of the README so each pinned number is wrong, and require the identical diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingCollectorRunner.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingCollectorRunner.cs index 531001642f..2bf7a4defa 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingCollectorRunner.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingCollectorRunner.cs @@ -1096,8 +1096,8 @@ items WERE found and merely some of their probes failed. Lite's twin is _lastCol /* The engine's provider, resolved ONCE for both branches. It used to be resolved only inside the per-database branch, and the branch below opened a hardcoded SqlConnection — so every collector that does NOT fan out per database was handed a SQL Server connection whatever the target was. - Six of the seven PostgreSQL collectors take that path (only pg_autovacuum_stats fans out), and - SqlClient rejects Npgsql's keywords while parsing the connection string, before any query runs: + Six of the seven PostgreSQL collectors then shipping took that path (only pg_autovacuum_stats + fanned out), and SqlClient rejects Npgsql's keywords while parsing the connection string, before any query runs: "Keyword not supported: 'host'". Worse, an ArgumentException is neither SqlException nor PostgresException, so it missed BOTH classification arms in DarlingWorker and recorded a raw ERROR every sweep forever — including for all three Tier 0 outage predictors. */ @@ -4627,8 +4627,8 @@ internal async Task OpenAzureDatabaseConnectionAsync(ServerRuntim /// The connection for a collector that reads the server as a whole — engine-resolved from the probed /// target, never constructed directly. /// Extracted so it can be PINNED by test. This is the exact seam that broke: the non-per-database - /// branch built a SqlConnection literally, so six of the seven PostgreSQL collectors got a SQL - /// Server connection and failed in the connection-string parser before running a query. Both engines' + /// branch built a SqlConnection literally, so six of the seven PostgreSQL collectors then + /// shipping got a SQL Server connection and failed in the connection-string parser before running a query. Both engines' /// providers were already correct and individually tested — nothing asserted that the RUNNER asked them. /// A test that opens nothing and only checks the returned TYPE is enough to catch it, which is why it is /// worth having. diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgStatementTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgStatementTools.cs index 0b84241d7d..734b7aef15 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgStatementTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgStatementTools.cs @@ -27,7 +27,7 @@ namespace PerformanceMonitor.Darling.Service.Mcp; [McpServerToolType] public sealed class DarlingMcpPgStatementTools { - [McpServerTool(Name = "get_pg_top_queries"), Description("Gets the top PostgreSQL query shapes by total execution time over a time period, for Amazon Aurora PostgreSQL targets. Includes Aurora's I/O source breakdown, which stock PostgreSQL cannot provide: a block 'read' may have come from the distributed storage volume or from the local NVMe Optimized Reads cache, and the two have very different costs. Also reports peak memory per statement, the closest PostgreSQL equivalent of a memory grant, and WAL bytes generated, which has no SQL Server DMV counterpart. Returns query_text for each statement, captured hourly and keyed on queryid, or null when none has been captured yet (a statement first seen minutes ago, or a queryid minted by a major-version upgrade). queryid itself is stable within a major version but changes across a major upgrade — which is exactly why the text is STORED rather than fetched live: after an upgrade the live view no longer holds the old ids, so their text would otherwise be unrecoverable and the history would read as a list of integers. queryid is returned as a STRING, not a number: it is a signed 64-bit value spread over the whole int8 range, so most ids exceed what a JSON number survives and a numeric wire form would be silently rounded by any parser that decodes numbers as IEEE-754 doubles — after which it matches nothing. Compare it and join on it as text. This is a separate tool from get_top_queries_by_cpu, which covers SQL Server.")] + [McpServerTool(Name = "get_pg_top_queries"), Description("Gets the top PostgreSQL query shapes by total execution time over a time period, wherever pg_stat_statements is installed: on Amazon Aurora the collector reads aurora_stat_statements(), and on any other PostgreSQL the vanilla view, with the Aurora-only columns null rather than zero. On Aurora that includes the I/O source breakdown, which stock PostgreSQL cannot provide: a block 'read' may have come from the distributed storage volume or from the local NVMe Optimized Reads cache, and the two have very different costs. Also reports peak memory per statement (Aurora only), the closest PostgreSQL equivalent of a memory grant, and WAL bytes generated, which has no SQL Server DMV counterpart. Returns query_text for each statement, captured hourly and keyed on queryid, or null when none has been captured yet (a statement first seen minutes ago, or a queryid minted by a major-version upgrade). queryid itself is stable within a major version but changes across a major upgrade — which is exactly why the text is STORED rather than fetched live: after an upgrade the live view no longer holds the old ids, so their text would otherwise be unrecoverable and the history would read as a list of integers. queryid is returned as a STRING, not a number: it is a signed 64-bit value spread over the whole int8 range, so most ids exceed what a JSON number survives and a numeric wire form would be silently rounded by any parser that decodes numbers as IEEE-754 doubles — after which it matches nothing. Compare it and join on it as text. This is a separate tool from get_top_queries_by_cpu, which covers SQL Server.")] public static async Task GetPgTopQueries( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, diff --git a/Darling/README.md b/Darling/README.md index 5ed6e256c9..c92968455b 100644 --- a/Darling/README.md +++ b/Darling/README.md @@ -694,8 +694,23 @@ GRANT pg_read_all_data TO darling_monitor; -- PostgreSQL 14+, see below | `pg_index_usage_stats` | `pg_stat_user_indexes`, `pg_statio_user_indexes`, `pg_index` | 24 h / 90 d | Per-index scan counts, sizes, and the constraint/replica-identity/validity facts that decide droppability. **Writers only, and per database** — a replica reports its OWN scan counts rather than the writer's, so an index that is load-bearing on the primary reads as unused there, which is the single worst answer this surface can give. `last_idx_scan` is PostgreSQL 16+ and is substituted with a typed NULL below it so the row shape does not change across a mixed-version fleet. Daily rather than hourly, matching `index_object_stats`: "has anything scanned this index" is a structural question, and an hourly sample would record the same catalog facts 24 times a day at 24x the fan-out connections. 90 days of retention because the RETENTION WINDOW IS THE EVIDENCE — an index can only be called unused for as long as we have been watching it, and 30 days cannot clear a monthly report. Indexes below 64 KB are not collected (nothing to model), except invalid ones, which are a finding at any size | | `pg_table_bloat_stats` | `pg_class`, `pg_stats`, `pg_stat_user_tables` | 60 min / 90 d | The statistics-based per-table bloat ESTIMATE, plus measured heap/TOAST/index sizes and the dead-tuple counts. **Never reads the relation**, which is what makes it affordable on a cadence: measured at 44 ms for a 2,001-table database against 860 ms for `pgstattuple` over the same tables, and that 19x is a floor on the ratio because those tables were tiny and fully cached. Hourly to match `pg_autovacuum_stats` deliberately — this measures the DAMAGE whose CAUSE that one measures, and correlating them needs a common grain. **Writers only, and per database.** Tables below 1 MB are not collected. The estimate is suppressed rather than captioned when its inputs cannot be trusted; see V85 | | `pg_session_states` | `pg_stat_activity` | 1 min / 30 d | Which sessions are holding a transaction open, for how long, and — the part that decides whether it matters — whether that transaction is what pins the xmin horizon. **`idle in transaction` is NOT automatically a horizon holder**, which is the whole reason this stores an age rather than a state: measured on live PostgreSQL 16.15, a READ COMMITTED transaction that only read and one whose `UPDATE` matched zero rows both sit idle in transaction indefinitely with `backend_xmin` AND `backend_xid` NULL, pinning nothing at all. `horizon_age` is the GREATER of the two ages and is `-1` when the session holds neither — not `0`, which would read as "holds the newest possible xid". **No raw statement text is stored**: `pg_stat_activity.query` carries literal parameter values, so the normalised `query_id` and a whitelisted command keyword are stored instead. **A SAMPLE, like `pg_blocking`** — a transaction that opened and closed between two cycles is invisible, and the read carries its own capture counts so that is visible rather than assumed. Valid on a standby, deliberately: a standby holds its own transactions, and with `hot_standby_feedback` on its xmin propagates to the primary | - -The two Aurora-only collectors — `pg_wait_stats` and `pg_cpu_utilization`, the Performance Insights CPU read that is not in the table above — are gated on Aurora detection, not on configuration: the connect probe looks for `aurora_version()` and the gate follows what it finds. Point Darling at self-managed PostgreSQL and the other 25 collectors run while those two sit out. +| `pg_server_config` | `pg_settings` | 60 min / 365 d | The configuration history — nothing else in the stack can reconstruct one that was never recorded. Hourly because configuration changes when a person changes it, and a year because "this got slow sometime last quarter" is asked long after the fact; `get_pg_server_config_changes` reads consecutive snapshots, so the change and its timing survive the person who made it. `source` is stored so the read can state which rows are server configuration and which are the collector's own session | +| `pg_deadlocks` | the server log via `pg_read_file()`; the RDS log API on Aurora/RDS | 5 min / 90 d | The full deadlock reports PostgreSQL writes to its log — previously only the COUNT (`pg_stat_database.deadlocks`) was collected, a number that goes up. `deadlock_hash` is the identity: the self-hosted route re-reads an overlapping log tail deliberately, so one deadlock lands many times and every read groups on the hash. `graph_text` is stored whole beside the parsed columns, so a lock shape the parser does not break out still arrives readable by a person. Needs nothing enabled at default settings, but `log_error_verbosity = terse` and a non-English `lc_messages` both silently empty it — the readiness collector's `message_locale` facet judges the latter | +| `pg_plan_capture` | `auto_explain` JSON in the server log via `pg_read_file()`; the RDS log API on Aurora/RDS | 60 min / 14 d | Captured execution plans, redacted before storage: query text is dropped outright (identity is `query_id`, whose normalised text `pg_statement_stats` carries) and every remaining string literal is scrubbed, with `plan_hash` taken of the REDACTED plan so a shape recurs to one hash whatever values it ran with. The self-hosted read is a fixed 4 MB tail — #2565 measured 772 MB of log in twenty seconds at capture-everything, so no fixed number bounds the worst case — and a server logging faster than the window covers should lower the cadence per server | +| `pg_plan_capture_readiness` | `pg_settings` (the `auto_explain` and logging GUCs) | 60 min / 365 d | Whether the target could capture plans at all, one row per facet in causal order — could it run `auto_explain`, is it loaded, is it capturing, in what format, and can a plan be attributed to a statement — because "no plans" has several unrelated causes and each carries its own remedy on the row. Hourly since every facet is a parameter-group setting; a year because "when did plan capture get turned on" is asked months later | +| `pg_write_stats` | `pg_stat_checkpointer`, `pg_stat_bgwriter`, `pg_stat_wal` | 1 min / 30 d | Checkpoints, background writing and WAL as ONE collector because they are one story: a high share of REQUESTED checkpoints means WAL filled `max_wal_size` before the interval elapsed. A union schema with version-conditional columns — PostgreSQL 17 moved the checkpoint counters out of `pg_stat_bgwriter`, so a major that lacks a column reports NULL rather than the collector splitting into per-version twins. PostgreSQL 14+ (`pg_stat_wal`); all three views are cluster-wide singletons, so a snapshot is one row | +| `pg_lock_stats` | `pg_locks` | 1 min / 30 d | Lock state by type, mode and relation, granted or waiting. Does NOT duplicate `pg_blocking_edges`, which stores blocked/blocker PAIRS — that answers who is stuck behind whom, and cannot answer what lock, on what object, is being waited on. Aggregated by (database, locktype, mode, granted, relation), so row count is bounded by CONTENTION rather than concurrency — and it is a SAMPLE, so the cadence is the resolution | +| `pg_replication_stats` | `pg_stat_replication` | 1 min / 30 d | The CONNECTED standbys — four byte distances from `pg_current_wal_lsn()` and three time lags. The counterpart of the slot collector and a different question: a slot describes what the primary RETAINS and exists even when nothing is attached (the disk-filling case); this describes replicas actually attached, and one that disappears here while its slot persists is the dangerous combination. The lag columns keep growing during a stall rather than freezing, which is what makes them usable as an alerting signal | +| `pg_buffer_usage` | `pg_buffercache` | 60 min / 30 d | What is resident in the shared buffer pool, per relation — which objects the cache is SPENT on, a different question from which are read most. Joins `pg_buffercache.relfilenode` to `pg_relation_filenode(c.oid)`, **not** to `c.oid`, which is the join every published example gets wrong, and scopes to the connected database because the pool is cluster-wide while `pg_class` is not. Hourly: the view is a scan of every buffer (6.1 ms at 512 MB, linear to ~780 ms at 64 GB), and a server without the extension records a non-fatal skip naming the one-line `CREATE EXTENSION` fix | +| `pg_extension_availability` | `pg_available_extensions`, `pg_extension` | 24 h / 365 d | Four states, not a boolean — `installed`, `outdated`, `available`, `absent` — because `available` is a `CREATE EXTENSION` away and `absent` is not. Absence is derived against an enumerated roster; `auto_explain` and `pg_wait_sampling` are deliberately off it, being preload-only modules that never appear in `pg_available_extensions` even where they are loaded and working. Per database as of V95, because `installed`/`outdated` are per-database facts | +| `pg_wait_sampling` | the `pg_wait_sampling` extension's profile | 60 min / 30 d | Wait events attributed to the query that waited — wait analysis existed ONLY on Aurora before this, which was backwards, because self-hosted is where the extension story is richest. `sample_count` is a tally of observations, not a duration, and `profile_period_ms` travels beside it because the count is uninterpretable alone. Cluster-wide with no database attribution, deliberately: the profile carries none, and inventing one would be the scope error V95 removed elsewhere | +| `pg_kernel_stats` | `pg_stat_kcache` | 60 min / 30 d | The kernel's own numbers per `queryid` — user/system CPU, bytes that reached the DEVICE, page faults. `pg_stat_statements` reports elapsed time, which cannot separate a query that was WAITING from one burning processor; these compose with it to make that split. `exec_read_bytes = 0` does not mean nothing was read: a read served by the OS page cache is genuinely zero. Top-level statements only, because `track = 'all'` would double-count every function body on the server | +| `pg_predicate_stats` | `pg_qualstats` | 60 min / 30 d | Which columns are filtered on, how selectively, and where the planner's estimate was wrong. `pg_index_usage_stats` records indexes that were USED; this records predicates that were EVALUATED, including the ones with no index behind them — the index-candidate signal, invisible to everything else here. `sample_rate` is stored per row and is usually not 1 (the extension defaults to `1/max_connections`), so counts are never read as complete. Per database, because the OIDs its rows carry only resolve in their own database | +| `pg_column_stats` | `pg_stats` | 24 h / 365 d | The planner inputs behind a misestimate — `n_distinct`, `null_frac`, `avg_width`, `correlation`, and the frequency of the single most common value. The SHAPE of the skew, never the values: `most_common_vals` and `histogram_bounds` hold raw customer data and are deliberately not stored. Per database. Daily and kept a year because these move when ANALYZE runs and are asked about retrospectively. Needs `pg_read_all_data` — without it the collector succeeds and stores zero rows, the quiet failure the paragraph above exists for | +| `pg_index_bloat` | `pg_stats`, `pg_class`, `pg_index` | 24 h / 90 d | The complete btree census with no size floor, ESTIMATED from catalog statistics with no page reads since #3234, ranked by reclaimable bytes and never by percentage. A row with a `skipped_reason` has NO answer rather than a healthy one, and every row carries the `pgstatindex` call for its own index as the on-request exact route. **Writers only, and per database.** Daily to match `pg_index_usage_stats` — the two answer halves of one question: is this index earning its keep, and is it wasting space doing it | +| `pg_cpu_utilization` | AWS Performance Insights (`os.cpuUtilization`) | 5 min / 30 d | Instance-level CPU for an Aurora target. This collector has no SQL route at all — PostgreSQL exposes no instance-level CPU signal — so every row arrives through the RDS/Performance Insights API, authorized by the monitoring host's AWS identity rather than any database grant. Five minutes rather than one because each cycle is an outbound `GetResourceMetrics` call per server, and each call asks for the last several 1-minute points, so a missed cycle backfills rather than gaps | + +The two Aurora-only collectors — `pg_wait_stats` and `pg_cpu_utilization`, the Performance Insights CPU read — are gated on Aurora detection, not on configuration: the connect probe looks for `aurora_version()` and the gate follows what it finds. Point Darling at self-managed PostgreSQL and the other 25 collectors run while those two sit out. `pg_autovacuum_stats` additionally gates OFF on a standby (`pg_is_in_recovery()`), and the reason is worth knowing because it is not a permissions or availability problem. `pg_stat_user_tables` reads fine on a replica and reports **all zeros**: measured on Aurora 17.7, the same cluster, database and 15 tables, the writer reported 13,654,458 dead tuples and 150,790,506 live tuples while the reader reported 0 for every tuple counter. Those are the writer's stats-collector numbers and they are not replicated. Ungated, a replica target would return no rows, the activity filter would read that as "nothing has pending work", and you would get a confident report of perfect autovacuum health for a cluster 13 million dead tuples behind. For the same reason, treat an empty `get_pg_replication_slots` result from a replica as per-instance rather than cluster-wide — slots live on the writer. @@ -725,7 +740,7 @@ The notable rungs are below. For the **complete** current schema, read `Darling/ | Version | Contents | |---|---| -| **V1** — collector tables | One table per collector, all 69, generated from the shared collector definitions (column-for-column identical to Lite's DuckDB schema): `wait_stats`, `latch_stats`, `spinlock_stats`, `query_stats`, `procedure_stats`, `query_store_stats`, `query_snapshots`, `plan_cache_stats`, `cpu_utilization_stats`, `cpu_scheduler_stats`, `file_io_stats`, `memory_stats`, `memory_clerks`, `memory_pressure_events`, `tempdb_stats`, `perfmon_stats`, `deadlocks`, `blocked_process_reports`, `dmv_blocking_snapshots`, `memory_grant_stats`, `waiting_tasks`, `session_stats`, `session_summary_stats`, `running_jobs`, `database_size_stats`, `index_object_stats`, `server_properties`, `system_health_events`, the four config snapshots (`server_config`, `database_config`, `database_scoped_config`, `trace_flags`), and the twenty PostgreSQL tables listed under V63–V69, V71, V83–V94 below | +| **V1** — collector tables | One table per collector, all 69, generated from the shared collector definitions (column-for-column identical to Lite's DuckDB schema): `wait_stats`, `latch_stats`, `spinlock_stats`, `query_stats`, `procedure_stats`, `query_store_stats`, `query_snapshots`, `plan_cache_stats`, `cpu_utilization_stats`, `cpu_scheduler_stats`, `file_io_stats`, `memory_stats`, `memory_clerks`, `memory_pressure_events`, `tempdb_stats`, `perfmon_stats`, `deadlocks`, `blocked_process_reports`, `dmv_blocking_snapshots`, `memory_grant_stats`, `waiting_tasks`, `session_stats`, `session_summary_stats`, `running_jobs`, `database_size_stats`, `index_object_stats`, `server_properties`, `system_health_events`, the four config snapshots (`server_config`, `database_config`, `database_scoped_config`, `trace_flags`), and the 27 PostgreSQL tables listed under V63–V69, V71, V83–V94, V96–V99, V102, V103 and V106 below | | **V2** — observability | `servers` (registry, upserted on every successful connect: identity, display name, engine edition, major version) and `collection_log` (one row per collector run: SUCCESS / PERMISSIONS / ERROR, row count, SQL-phase and storage-phase timings) | | **V3** — alerting | `config_alert_log` (one history row per fired alert), `config_edge_trigger_watermarks` (restart-surviving edge-trigger and failed-job watermarks), `config_mute_rules` (alert mute rules; starts empty) | | **V4** — analysis | `analysis_findings` (persisted findings incl. the stored remediation action), `analysis_muted` (muted finding patterns), and 17 `v_` passthrough views so the shared analysis SQL runs verbatim against this store | @@ -775,6 +790,13 @@ The notable rungs are below. For the **complete** current schema, read `Darling/ | **V93** — PostgreSQL buffer residency | `collect.pg_buffer_usage` + its time index (#2544): what is resident in shared buffers, by relation — a hit ratio says how often the pool worked, this says what is IN it. Joins `pg_buffercache.relfilenode` to `pg_relation_filenode(c.oid)`, **not** to `c.oid`, which is the join every published example gets wrong, and scopes to the connected database because the pool is cluster-wide while `pg_class` is not | | **V94** — PostgreSQL index bloat | `collect.pg_index_bloat` + its time index (#2561): b-tree index bloat, measured via `pgstatindex` **as this rung shipped it**. That decision was correct on its own terms and #3234 reversed it: the estimator route was blind because it needs `pg_stats`, which returns nothing to a `pg_monitor` role, and once `pg_read_all_data` was granted the estimator became viable and the census became the on-request path. Read this row as history — the current behaviour is the grant and extension story above. `avg_leaf_density` is stored RAW and never converted to a percentage, and `skipped_reason` exists so a size ceiling can never masquerade as an absence of bloat | | **V95** — PostgreSQL per-database attribution | `database_name` on `collect.pg_column_stats`, `collect.pg_index_bloat` and `collect.pg_extension_availability` (#2599), and `pg_extension_availability` now runs per database. Two of these collectors ran per database and could not say WHICH database a row described, so on a cluster carrying one schema in two databases their rows collided; the third read the per-database `pg_extension` catalog through a single connection and reported one database's answer for the whole server. Nullable with no backfill — a catalog-only change in PostgreSQL, and rows collected before this rung genuinely do not know where they came from | +| **V96** — PostgreSQL wait sampling | `collect.pg_wait_sampling` + its time index (#2603): wait events attributed to the query that waited, from the `pg_wait_sampling` extension. Wait analysis existed ONLY on Aurora before this — backwards, because self-hosted is where the extension story is richest. `sample_count` is a tally of observations, not a duration, and `profile_period_ms` travels beside it because the count is uninterpretable alone; storing a derived millisecond figure would bake today's sampling period into history that outlives it. No `database_name`, deliberately: the profile is cluster-wide, and attributing its rows to a database would be exactly the scope error V95 removed from three other tables | +| **V97** — PostgreSQL kernel stats | `collect.pg_kernel_stats` + its time index (#2603): the operating system's own CPU and disk numbers per `queryid`, from `pg_stat_kcache`. `pg_stat_statements` reports elapsed time, which cannot separate a query that was WAITING from one that was burning processor — these compose with it to make that split. `exec_read_bytes` measures I/O that reached the DEVICE (a page-cache hit is genuinely zero), and `stats_since` is stored so a reset is a recorded fact rather than something inferred from a counter that went backwards | +| **V98** — PostgreSQL predicate stats | `collect.pg_predicate_stats` + its time index (#2603): which columns are filtered on, how selectively, and where the planner's estimate was wrong, from `pg_qualstats`. `pg_index_usage_stats` records indexes that were USED; this records predicates that were EVALUATED, including the ones with no index behind them — the index-candidate signal, invisible to everything else here. `sample_rate` is stored per row and is usually not 1 (the extension defaults to `1/max_connections`), so counts can never be read as complete; `worst_estimate_error_ratio` is the reason to look — selectivity says an index might help, the error ratio says the planner does not understand the column | +| **V99** — PostgreSQL plan capture | `collect.pg_plan_capture` + its time index (#2566): `auto_explain`'s JSON plans out of the server log. Its own table, NOT `query_plan_dim` — that one is a documented XML contract with direct-SQL consumers, and JSON rows would break their dashboards with no schema change to notice. No query text and no literals reach the store: identity is `query_id`, every remaining string is redacted before storage, and `plan_hash` is of the REDACTED plan so a shape recurs to one hash whatever values it ran with. Self-hosted reads the log with `pg_read_file`; Aurora and RDS have no filesystem and take the RDS log API instead (#2538) | +| **V102** — PostgreSQL server configuration | `collect.pg_server_config` + its time index (#2658): every `pg_settings` row, hourly for a year — SQL Server answers this three ways and PostgreSQL had no answer at all, and no other part of the stack can reconstruct a configuration history that was never recorded. Nothing is filtered at collection: `source` is stored so the READ can state which rows are server configuration and which are the collector's own session, where the rule can be enforced and explained | +| **V103/V104** — PostgreSQL deadlocks | `collect.pg_deadlocks` + its time index, then the `deadlock_hash` lookup index as its own rung (#2661): the full reports PostgreSQL writes to its log — we collected the COUNT (`pg_stat_database.deadlocks`) and nothing else. `deadlock_hash` is the identity, because the `pg_read_file` route re-reads an OVERLAPPING tail deliberately so one deadlock lands many times, and every read groups on the hash to answer once per deadlock. `graph_text` is stored whole beside the parsed columns, so a lock shape the parser does not recognise still arrives readable by a person. The index is a separate rung because the schema-generator pin permits exactly one index per collector rung — split out, BOTH fresh and upgraded stores get it | +| **V106** — PostgreSQL instance CPU | `collect.pg_cpu_utilization` + its time index (#2719): instance-level CPU for an Aurora target, from the Performance Insights API. This collector has no SQL route at all — PostgreSQL exposes no instance-level CPU signal — so every row arrives through the monitoring host's AWS identity, never a database connection | | **V72** — Query Store plan map | `collect.query_store_plan_map` — `(server_id, database_name, plan_id)` → digest, so Query Store facts can reference plan XML they no longer carry once that content moves into the shared `query_plan_dim`. Plan XML was stored INLINE on `query_store_stats` at roughly 5x redundancy. Not a hypertable: one row per distinct plan per database, so it is dimension-shaped and pruned on `last_seen` rather than by `drop_chunks`. Its `last_seen` is load-bearing — the dimension GC sweeps on timestamps rather than counting references, so ending the re-shipping also ends the liveness signal that used to keep those dim rows alive | | **V73** — PostgreSQL statement text | `collect.pg_statement_text` — `(server_id, queryid)` → statement text, refreshed hourly, so `get_pg_top_queries` returns something readable (#2219). `pg_statement_stats` stores no text because `showtext` is a real per-collection cost and normalized text is highly repetitive; but `queryid` is NOT stable across a major version upgrade, so without this the stored history joins to nothing after one — a list of integers that used to be your slowest queries, unrecoverable because the live view no longer holds the old ids. Text is INLINE rather than a `query_text_dim` digest: the dimension route needs the GC liveness interlock whose failure mode is silently missing text, and inline cannot dangle. Not a hypertable and not a collector table, exactly like V72 — a bespoke upsert path, pruned on `last_seen` with a margin that makes text OUTLIVE the statistics referencing it | | **V76** — Query Store health | `collect.query_store_health` + its index + the `v_query_store_health` passthrough view — the #2319 per-database `sys.database_query_store_options` collector's store table: actual vs desired state (the cap-hit READ_ONLY transition and its readonly_reason), current vs max storage, cleanup thresholds, and the runtime-stats interval length. A fresh store gets the table from V1's generated schema; V76 is what an already-existing store gets | diff --git a/Lite.Tests/AuroraOnlySqlIsGatedTests.cs b/Lite.Tests/AuroraOnlySqlIsGatedTests.cs index 7b169f2ed5..0ed2f43870 100644 --- a/Lite.Tests/AuroraOnlySqlIsGatedTests.cs +++ b/Lite.Tests/AuroraOnlySqlIsGatedTests.cs @@ -78,6 +78,12 @@ public class AuroraOnlySqlIsGatedTests ["CollectorEngineCapability.cs"] = "PROSE. Names the surfaces in the sentence shown to an operator when a collector cannot run here. " + "It describes the dependency rather than depending on it.", + + ["DarlingMcpPgStatementTools.cs"] = + "PROSE (#3261). get_pg_top_queries' description states the #2625 dual-source truth — Aurora reads " + + "aurora_stat_statements(), every other PostgreSQL the vanilla view with the Aurora-only columns " + + "null. The tool reads the STORE, never the target; the dependency itself is " + + "PgStatementStatsCollector's PAIRED entry above.", }; [Fact]