diff --git a/Darling/Darling.Tests/DarlingMcpStoreMetricsToolsTests.cs b/Darling/Darling.Tests/DarlingMcpStoreMetricsToolsTests.cs index 45b6de53d..7ed70b159 100644 --- a/Darling/Darling.Tests/DarlingMcpStoreMetricsToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpStoreMetricsToolsTests.cs @@ -1420,6 +1420,23 @@ public async Task TheLiveInventoryReads_FailToNull_AndSkipTheCatalogsThatDoNotEx Assert.Null(await DarlingStoreMetricsReader.GetLargestUnenumeratedAsync(nowhere, notRegistered, ct)); } + /// + /// #4758: the description's requested field says what PostgreSQL's counter counts. A requested + /// checkpoint is started by WAL volume reaching max_wal_size, a base backup or a CHECKPOINT + /// statement, and the counter does not say which, so the description must not call them all WAL-forced. + /// + [Fact] + public void TheDescription_SaysARequestedCheckpointHasThreeCauses_NotOnlyWalVolume() + { + var description = ToolMethods().Single().GetCustomAttribute()?.Description; + Assert.NotNull(description); + + Assert.Contains( + "requested (checkpoints started by WAL volume reaching max_wal_size, a base backup, or a CHECKPOINT statement rather than by checkpoint_timeout; the counter does not say which)", + description!, StringComparison.Ordinal); + Assert.DoesNotContain("checkpoints forced by WAL volume", description!, StringComparison.Ordinal); + } + /// /// #3582: the description names every new kind, the coverage fields and the reconciliation bar, /// asserted TOGETHER with the sweep arms that write the kinds and the constants that set the bar. diff --git a/Darling/Darling.Tests/StoreSelfMetricsTests.cs b/Darling/Darling.Tests/StoreSelfMetricsTests.cs index ec4f13506..8623c1b73 100644 --- a/Darling/Darling.Tests/StoreSelfMetricsTests.cs +++ b/Darling/Darling.Tests/StoreSelfMetricsTests.cs @@ -884,7 +884,7 @@ store rather than a difference of two numbers that happened to be there. */ Assert.True(second.SyncMs is >= 0); Assert.True(second.Requested is >= 1, $"the forced CHECKPOINT must land in the interval's requested count, not {second.Requested}"); Assert.Equal(first.CumulativeRequested + second.Requested, second.CumulativeRequested); - Assert.True(second.IsPressure, "one WAL-forced checkpoint in the interval IS the pressure arm"); + Assert.True(second.IsPressure, "one requested checkpoint in the interval IS the pressure arm"); /* #4619: the existence check, through the shipped SQL, over the names THIS SWEEP WROTE. Every one must resolve back to its object: a kind whose lookup drifted from the sweep's name form (the job name, a diff --git a/Darling/Darling.Tests/StoreToastAndCheckpointerTests.cs b/Darling/Darling.Tests/StoreToastAndCheckpointerTests.cs index 01f968c4c..18783c377 100644 --- a/Darling/Darling.Tests/StoreToastAndCheckpointerTests.cs +++ b/Darling/Darling.Tests/StoreToastAndCheckpointerTests.cs @@ -588,9 +588,17 @@ rule would have judged 84s against the same bar and misfired (#4037). */ Assert.Contains("a per-checkpoint sync average past 10s", storm, StringComparison.Ordinal); Assert.Contains("25.2 s and 14.0 s sync phases", storm, StringComparison.Ordinal); - /* Requested arm: low sync average, but one requested checkpoint still fires. */ + /* Requested arm: low sync average, but one requested checkpoint still fires. The note says what the alert + says (#4758): a requested checkpoint comes from WAL volume, a base backup or a CHECKPOINT statement, the + counters do not say which, and raising max_wal_size is the remedy for the first cause only. */ var forced = DarlingMcpStoreMetricsTools.CheckpointerNote(DarlingStoreMetricsReader.CheckpointerReading.From(new(t1, 2_000, 3_000, 6, Timed: 5), new(t0, 1_000, 1_000, 5, Timed: 0))); - Assert.Contains("a requested checkpoint means the store wrote more WAL between checkpoints than max_wal_size allows", forced, StringComparison.Ordinal); + Assert.Contains("a requested checkpoint comes from WAL volume reaching max_wal_size, a base backup, or a CHECKPOINT statement", forced, StringComparison.Ordinal); + Assert.Contains("counters do not say which", forced, StringComparison.Ordinal); + Assert.Contains("log_checkpoints", forced, StringComparison.Ordinal); + Assert.Contains("Raising max_wal_size is the remedy only for the first cause", forced, StringComparison.Ordinal); + Assert.DoesNotContain("WAL-forced", forced, StringComparison.Ordinal); + Assert.DoesNotContain("means the store wrote more WAL", forced, StringComparison.Ordinal); + Assert.Contains("The informational Store Checkpointer Pressure self-alert says the same.", forced, StringComparison.Ordinal); /* Zero checkpoints in the interval: no average to state, unmeasured, not judged either way. */ var idle = DarlingMcpStoreMetricsTools.CheckpointerNote(DarlingStoreMetricsReader.CheckpointerReading.From(new(t1, 1_000, 1_000, 5, Timed: 0), new(t0, 1_000, 1_000, 5, Timed: 0))); @@ -899,12 +907,25 @@ public async Task CheckpointerPressure_FiresInfoOnEitherArm_AndNotUnderBothLines Assert.Equal("average sync > 10s per checkpoint, or any requested checkpoint", fired.ThresholdValue); Assert.Contains("25.2 s and 14.0 s sync phases", fired.DetailText!, StringComparison.Ordinal); - /* The other arm alone: one WAL-forced checkpoint with a short average sync. */ + /* The other arm alone: one requested checkpoint with a short average sync. A requested checkpoint comes + from WAL volume reaching max_wal_size, a base backup, or a CHECKPOINT statement, and the counters do not + say which (#4758), so the alert calls it "requested", names all three causes and the log_checkpoints + lines that say which, and offers max_wal_size as the remedy for the first cause only. */ var forced = new Harness(); await forced.Build().ApplyCheckpointerPressureAsync(Interval(syncMs: 800, requested: 1, timed: 1), Ct); var forcedFire = Assert.Single(forced.Deliverer.Outcomes); - Assert.Contains("1 WAL-forced checkpoint(s)", forcedFire.CurrentValue, StringComparison.Ordinal); - Assert.Contains("more WAL between checkpoints than max_wal_size allows", forcedFire.DetailText!, StringComparison.Ordinal); + Assert.Contains("1 requested checkpoint(s)", forcedFire.CurrentValue, StringComparison.Ordinal); + Assert.DoesNotContain("WAL-forced", forcedFire.CurrentValue, StringComparison.Ordinal); + Assert.DoesNotContain("WAL-forced", forcedFire.ShortMessage!, StringComparison.Ordinal); + Assert.Contains("WAL volume reaching max_wal_size, a base backup, or a CHECKPOINT statement", forcedFire.DetailText!, StringComparison.Ordinal); + Assert.Contains("base backup", forcedFire.DetailText!, StringComparison.Ordinal); + Assert.Contains("CHECKPOINT statement", forcedFire.DetailText!, StringComparison.Ordinal); + Assert.Contains("counters do not say which", forcedFire.DetailText!, StringComparison.Ordinal); + Assert.Contains("log_checkpoints", forcedFire.DetailText!, StringComparison.Ordinal); + Assert.Contains("Raising max_wal_size is the remedy only for the first cause", forcedFire.DetailText!, StringComparison.Ordinal); + Assert.DoesNotContain("forced by WAL volume", forcedFire.DetailText!, StringComparison.Ordinal); + Assert.Null(forcedFire.Severity); + Assert.Equal("average sync > 10s per checkpoint, or any requested checkpoint", forcedFire.ThresholdValue); /* Exactly the bar, no forced checkpoint: quiet. */ var quiet = new Harness(); @@ -1013,7 +1034,7 @@ public async Task CheckpointerPressure_TheSameCountersWithoutARestart_StillFireA await h.Build().ApplyCheckpointerPressureAsync(Interval(syncMs: 800, requested: 2, restarted: false, timed: 1), Ct); var fired = Assert.Single(h.Deliverer.Outcomes); Assert.Equal(DarlingSelfAlertEvaluator.CheckpointerPressureMetric, fired.MetricName); - Assert.Contains("2 WAL-forced checkpoint(s)", fired.CurrentValue, StringComparison.Ordinal); + Assert.Contains("2 requested checkpoint(s)", fired.CurrentValue, StringComparison.Ordinal); Assert.DoesNotContain("restart", fired.ShortMessage!, StringComparison.OrdinalIgnoreCase); Assert.DoesNotContain("RESTARTED", fired.DetailText!, StringComparison.Ordinal); } @@ -1040,7 +1061,7 @@ public async Task CheckpointerPressure_TheSameSlowSyncWithoutARestart_StillFires await h.Build().ApplyCheckpointerPressureAsync(Interval(syncMs: 25_200, requested: 1, restarted: false, timed: 1), Ct); var fired = Assert.Single(h.Deliverer.Outcomes); Assert.Contains("average sync", fired.CurrentValue, StringComparison.Ordinal); - Assert.Contains("1 WAL-forced checkpoint(s)", fired.CurrentValue, StringComparison.Ordinal); + Assert.Contains("1 requested checkpoint(s)", fired.CurrentValue, StringComparison.Ordinal); Assert.DoesNotContain("restart", fired.ShortMessage!, StringComparison.OrdinalIgnoreCase); } diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingSelfAlertEvaluator.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingSelfAlertEvaluator.cs index fac77a33d..d1339df0c 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingSelfAlertEvaluator.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingSelfAlertEvaluator.cs @@ -966,7 +966,9 @@ by dimension table name (two today: query_text_dim, query_plan_dim), STANDING li /// checkpoint_timeout covers about twelve timed checkpoints, so judging the sum against a /// per-checkpoint bar breached on a healthy store whose checkpoints each synced five to eight seconds and /// never recovered (#4037's own measured population). The second arm, checkpoints_requested > 0, - /// has no threshold to tune: one WAL-forced checkpoint in an hour says the store outran max_wal_size. + /// has no threshold to tune: one requested checkpoint in an hour is worth a look, and it is a look, not a + /// verdict, because a requested checkpoint comes from WAL volume reaching max_wal_size, a base backup, + /// or a CHECKPOINT statement and the counters do not say which (#4758). /// Both arms are judged on an interval with no postmaster restart inside it: across one, the shutdown /// checkpoint is in the requested count and in the phase times alike, so neither is judged (#3955); and the /// average arm needs a TIMED count on both samples of the pair (V140), so a row from before that rung @@ -5851,10 +5853,12 @@ public async Task EvaluateCheckpointerPressureAsync(NpgsqlDataSource postgres, C /// /// Applies the fleet-level Store Checkpointer Pressure condition (#3783) from the store's last measured /// checkpointer interval: the sync (fsync) phase held more than inside the - /// interval, OR at least one checkpoint was REQUESTED — forced by WAL volume reaching max_wal_size - /// rather than by the clock. Three unattributed read kills on a production store in one day all sat inside - /// checkpoint sync phases of 25.2 s and 14.0 s with nothing recording that the checkpointer had been there; - /// this is that record, as an alert. A STANDING condition (the Store Job Over Cadence idiom): fire once on + /// interval, OR at least one checkpoint was REQUESTED — started by WAL volume reaching max_wal_size, a + /// base backup, or a CHECKPOINT statement rather than by the clock. PostgreSQL's counters do not record + /// which, only its log_checkpoints lines do, so the text names all three causes and offers raising + /// max_wal_size as the remedy for the first only (#4758). Three unattributed read kills on a + /// production store in one day all sat inside checkpoint sync phases of 25.2 s and 14.0 s with nothing + /// recording that the checkpointer had been there; this is that record, as an alert. A STANDING condition (the Store Job Over Cadence idiom): fire once on /// breach, re-fire on the shared alert cooldown while each new interval keeps breaching, one "Store /// Checkpointer Pressure Recovered" resolution when an interval reads clean. INFORMATIONAL, fired with no /// severity override: the two levers are configuration — #3802's WAL sizing (max_wal_size / @@ -5871,7 +5875,7 @@ public async Task EvaluateCheckpointerPressureAsync(NpgsqlDataSource postgres, C /// /// An interval that spans a postmaster restart judges neither arm (#3955). PostgreSQL counts the /// shutdown checkpoint as requested and keeps the count across the restart, so every service restart that - /// stopped the store fired this alert as "WAL-forced" pressure the store did not have; and that checkpoint's + /// stopped the store fired this alert as requested-checkpoint pressure the store did not have; and that checkpoint's /// own write and sync phases land in the same counters, where nothing can separate them from the live /// checkpoints' (a fast shutdown flushes every dirty buffer at once, with every client already gone, so a /// long one is not a stall anyone's read sat inside). The reader therefore calls such an interval @@ -5913,9 +5917,9 @@ internal async Task ApplyCheckpointerPressureAsync( var averageSecondsText = averageSyncMs is double a ? (a / 1000.0).ToString("0.0", CultureInfo.InvariantCulture) : "unmeasured"; var arms = (averageOverBar, requested > 0) switch { - (true, true) => $"average sync {averageSecondsText}s per checkpoint and {requested} WAL-forced checkpoint(s)", + (true, true) => $"average sync {averageSecondsText}s per checkpoint and {requested} requested checkpoint(s)", (true, false) => $"average sync {averageSecondsText}s per checkpoint", - _ => $"{requested} WAL-forced checkpoint(s)", + _ => $"{requested} requested checkpoint(s)", }; await FireAsync( StoreKey(CheckpointerKey), _storeLabel, CheckpointerPressureMetric, @@ -5924,15 +5928,17 @@ await FireAsync( detail: $"The store's own checkpointer ran {checkpointCount} checkpoint(s) over the {intervalMinutes} minutes " + $"between the last two self-metrics sweeps, spending {(syncMs / 1000.0).ToString("0.0", CultureInfo.InvariantCulture)}s " + $"total in its sync (fsync) phase and {writeSeconds}s in its write phase — an average of {averageSecondsText}s of sync " + - $"per checkpoint — and {requested} of those checkpoints were REQUESTED — forced by WAL volume reaching " + - "max_wal_size rather than by checkpoint_timeout. " + + $"per checkpoint — and {requested} of those checkpoints were REQUESTED — started by something other than " + + "checkpoint_timeout: WAL volume reaching max_wal_size, a base backup, or a CHECKPOINT statement. " + (averageOverBar ? $"A per-checkpoint sync average past {barSeconds}s means at least some of this interval's checkpoints ran " + "an I/O stall every reader on the store shares: on one production store, three read kills in a day that " + "nothing else explained all sat inside 25.2 s and 14.0 s sync phases, and the MCP host's read deadline is " + $"the {barSeconds}s this line is drawn at. " - : "A requested checkpoint means the store wrote more WAL between checkpoints than max_wal_size " + - "allows, so the checkpointer ran early and the next one is closer — checkpoint pressure compounds. ") + + : "PostgreSQL's counters do not say which of those started a requested checkpoint; the log_checkpoints " + + "lines do. Raising max_wal_size is the remedy only for the first cause: when WAL volume did it, the " + + "store wrote more WAL between checkpoints than max_wal_size allows, so the checkpointer ran early and " + + "the next one is closer — checkpoint pressure compounds. ") + "The interval series is collect.store_metrics (object_kind = 'checkpointer'; the stored columns are the " + "server's cumulative counters, and get_store_metrics' checkpointer block publishes the per-interval " + "differences and the same per-checkpoint average). This alert re-fires on the alert cooldown while each " + diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpStoreMetricsTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpStoreMetricsTools.cs index d0e56ac71..b28780fad 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpStoreMetricsTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpStoreMetricsTools.cs @@ -79,7 +79,7 @@ public sealed class DarlingMcpStoreMetricsTools + "answered. job_history is ownership-filtered: empty may mean no visibility, not no jobs; check " + "execution_logging/owner_evidence. retention.held policies are PAUSED by the rollup gate; arming by " + "hand drops history's only copy. inventory.reconciled=false flags bytes no row explains. <> " - + "Gets the monitoring store's OWN size and growth metrics — not a monitored SQL Server's. The service records an hourly self-metrics snapshot per store object: each hypertable, and each continuous aggregate (object_kind continuous_aggregate, sized through its materialization, because timescaledb_information.hypertables never lists a materialization), with total and pre/post-compression bytes and chunk count; the query-text and query-plan payload dimensions (object_kind dimension) with row counts; the named plain tables (object_kind table: collect.query_store_text, collect.query_store_plan_map, config.config_alert_log); two catch-alls with byte total and relation count, object_kind other (user-schema relations no named row accounts for) and object_kind system (catalogs and TimescaleDB bookkeeping); the whole store with the enabled-server count; and each TimescaleDB background job (object_kind background_job) with its last run duration, schedule interval, duration_vs_cadence_percent and run/failure totals. SUMMARY FIRST: by default it returns the store-level blocks (inventory, retention, job_history, checkpointer; below), the whole-store daily growth with the per-server ingest rate (daily growth / enabled servers, the number onboarding N servers multiplies), and three ranked lists, each bounded by limit with its match count and truncated: objects (the largest byte-bearing objects), fastest_growing (by growth_bytes over the window) and background_jobs (jobs whose failures grew in the window first, then by duration_vs_cadence_percent). Each row carries its change over the window from delta_since instead of a per-object series. object_kind lists every object of one kind; object_name drills in: an exact name returns that object's daily series over days_back, a partial name lists the objects whose names contain it. The inventory block RECONCILES the newest sweep against its own pg_database_size: enumerated_percent is the share under named objects, attributed_percent the share any row accounts for, residual_bytes what no row explains, and reconciled is false, a finding, when that residual exceeds the larger of 1% and 64 MiB. bytes_by_kind answers 'what is driving growth' in one glance, and largest_unenumerated names, live, the biggest relations inside other. Each continuous_aggregate row carries, live from the catalog, compression_enabled and its refresh, compression and retention policy job ids. The retention block lists, live, every retention policy the rollup-coverage gate holds PAUSED, with the tier's actual span and how many times its drop_after horizon it is holding: a held policy looks healthy in the stored job telemetry and is a common cause of unexplained growth. Each daily point is that day's LAST snapshot, never its maximum or its mean, so a MAXIMUM question (a job's longest run that day) cannot be answered from it: the day's peak is DROPPED rather than smoothed, and the hourly sample misses a run longer than its offset altogether. The per-run route is timescaledb_information.job_history, which records a SUCCESSFUL run only while timescaledb.enable_job_execution_logging is on, and that setting defaults OFF (a FAILED run's row is written regardless of it), so on a store that never turned it on a maximum over it covers zero rows of successful runs. The job_history block in every response reports that setting's EFFECTIVE value and source: read it before treating an empty job_history as an answer. Whether YOU see rows is a second question, because job_history is ownership-filtered — its rows are visible only to members of the job's owner role or of the database owner, while the jobs and job_stats views show every role every job. So the block reports which role it read as (reader_role), that role's standing under the view's own predicate (visibility: All, Partial or None), the rows it observed over a fixed 24-hour window (rows_observed, newest_row_at) beside the jobs job_stats says started a run in it (jobs_run_in_window), and contradiction, true only when recording is on, a reader admitted to every job's history saw no rows, and jobs ran. In managed mode this tool reads as the least-privilege mcp role, which the view filters out. The role that can see is the service's owner, and the service's hourly self-metrics sweep runs as it and persists the owner's own count over the same 24-hour window, reported as owner_evidence (Observed, Stale, Filtered or Absent), owner_role, owner_observed_at, owner_rows_observed, owner_newest_row_at and owner_jobs_run_in_window: a measurement taken by the service's sweep rather than by this connection. TOAST UTILISATION (V137): each dimension row also carries toast_bytes, toast_live_bytes, toast_utilisation_pct and toast_utilisation_note, because plan XML and statement text do not live in the dimension's heap — they live in its TOAST file, and pg_total_relation_size says how big that file is and nothing about how FULL it is. Row churn leaves free space in that file that ordinary VACUUM returns to the table and never to the operating system. toast_utilisation_pct is toast_live_bytes / toast_bytes (one decimal), null when either is null or the file is empty; toast_live_bytes needs the pg_freespacemap extension, which the bundled store image ships and does not install, and without it the note says 'not measured' and NOTHING is computed from total_bytes or a tuple share. Under 50 % on a file over 10 GiB the note carries the reclaim path: --recompress-plan-dim --vacuum-full, at the MAINTAINER's word in a maintenance window, because VACUUM FULL takes an ACCESS EXCLUSIVE lock on the dimension and needs free disk for a full copy of the live data; the informational Store TOAST Slack self-alert fires daily on the same condition. This tool never reclaims anything by itself. Hypertable, aggregate, table and catch-all objects carry the four TOAST fields as null. CHECKPOINTER (V137): the checkpointer block is the store's OWN checkpointer over the last self-metrics interval: write_ms and sync_ms (milliseconds in its write and sync/fsync phases), requested (checkpoints forced by WAL volume reaching max_wal_size rather than by checkpoint_timeout), interval_seconds (MEASURED between the two sweeps that bound it, not the assumed cadence), and the cumulative counters beside them. status says whether there IS an interval: Observed, NoPrevious (one row so far), Reset (a counter went backwards — a stats reset or restart between sweeps, so no delta is stated rather than a negative one), Restarted (the store's postmaster restarted between the two sweeps, postmaster_restarted true: PostgreSQL counts the shutdown checkpoint as requested and keeps the count across the restart, and that checkpoint's own write and sync phases land in the same counters, so no delta is stated and the self-alert judges neither arm; the next sweep's interval is judged normally), or Absent; postmaster_start_time is when the store's postmaster that produced the newest row started (null on a row older than V139). The informational Store Checkpointer Pressure self-alert fires when sync_ms exceeds 10,000 or requested exceeds 0 in an interval, naming WAL sizing and refresh slicing as the levers. The checkpointer row is never an object row: it carries no bytes.")] + + "Gets the monitoring store's OWN size and growth metrics — not a monitored SQL Server's. The service records an hourly self-metrics snapshot per store object: each hypertable, and each continuous aggregate (object_kind continuous_aggregate, sized through its materialization, because timescaledb_information.hypertables never lists a materialization), with total and pre/post-compression bytes and chunk count; the query-text and query-plan payload dimensions (object_kind dimension) with row counts; the named plain tables (object_kind table: collect.query_store_text, collect.query_store_plan_map, config.config_alert_log); two catch-alls with byte total and relation count, object_kind other (user-schema relations no named row accounts for) and object_kind system (catalogs and TimescaleDB bookkeeping); the whole store with the enabled-server count; and each TimescaleDB background job (object_kind background_job) with its last run duration, schedule interval, duration_vs_cadence_percent and run/failure totals. SUMMARY FIRST: by default it returns the store-level blocks (inventory, retention, job_history, checkpointer; below), the whole-store daily growth with the per-server ingest rate (daily growth / enabled servers, the number onboarding N servers multiplies), and three ranked lists, each bounded by limit with its match count and truncated: objects (the largest byte-bearing objects), fastest_growing (by growth_bytes over the window) and background_jobs (jobs whose failures grew in the window first, then by duration_vs_cadence_percent). Each row carries its change over the window from delta_since instead of a per-object series. object_kind lists every object of one kind; object_name drills in: an exact name returns that object's daily series over days_back, a partial name lists the objects whose names contain it. The inventory block RECONCILES the newest sweep against its own pg_database_size: enumerated_percent is the share under named objects, attributed_percent the share any row accounts for, residual_bytes what no row explains, and reconciled is false, a finding, when that residual exceeds the larger of 1% and 64 MiB. bytes_by_kind answers 'what is driving growth' in one glance, and largest_unenumerated names, live, the biggest relations inside other. Each continuous_aggregate row carries, live from the catalog, compression_enabled and its refresh, compression and retention policy job ids. The retention block lists, live, every retention policy the rollup-coverage gate holds PAUSED, with the tier's actual span and how many times its drop_after horizon it is holding: a held policy looks healthy in the stored job telemetry and is a common cause of unexplained growth. Each daily point is that day's LAST snapshot, never its maximum or its mean, so a MAXIMUM question (a job's longest run that day) cannot be answered from it: the day's peak is DROPPED rather than smoothed, and the hourly sample misses a run longer than its offset altogether. The per-run route is timescaledb_information.job_history, which records a SUCCESSFUL run only while timescaledb.enable_job_execution_logging is on, and that setting defaults OFF (a FAILED run's row is written regardless of it), so on a store that never turned it on a maximum over it covers zero rows of successful runs. The job_history block in every response reports that setting's EFFECTIVE value and source: read it before treating an empty job_history as an answer. Whether YOU see rows is a second question, because job_history is ownership-filtered — its rows are visible only to members of the job's owner role or of the database owner, while the jobs and job_stats views show every role every job. So the block reports which role it read as (reader_role), that role's standing under the view's own predicate (visibility: All, Partial or None), the rows it observed over a fixed 24-hour window (rows_observed, newest_row_at) beside the jobs job_stats says started a run in it (jobs_run_in_window), and contradiction, true only when recording is on, a reader admitted to every job's history saw no rows, and jobs ran. In managed mode this tool reads as the least-privilege mcp role, which the view filters out. The role that can see is the service's owner, and the service's hourly self-metrics sweep runs as it and persists the owner's own count over the same 24-hour window, reported as owner_evidence (Observed, Stale, Filtered or Absent), owner_role, owner_observed_at, owner_rows_observed, owner_newest_row_at and owner_jobs_run_in_window: a measurement taken by the service's sweep rather than by this connection. TOAST UTILISATION (V137): each dimension row also carries toast_bytes, toast_live_bytes, toast_utilisation_pct and toast_utilisation_note, because plan XML and statement text do not live in the dimension's heap — they live in its TOAST file, and pg_total_relation_size says how big that file is and nothing about how FULL it is. Row churn leaves free space in that file that ordinary VACUUM returns to the table and never to the operating system. toast_utilisation_pct is toast_live_bytes / toast_bytes (one decimal), null when either is null or the file is empty; toast_live_bytes needs the pg_freespacemap extension, which the bundled store image ships and does not install, and without it the note says 'not measured' and NOTHING is computed from total_bytes or a tuple share. Under 50 % on a file over 10 GiB the note carries the reclaim path: --recompress-plan-dim --vacuum-full, at the MAINTAINER's word in a maintenance window, because VACUUM FULL takes an ACCESS EXCLUSIVE lock on the dimension and needs free disk for a full copy of the live data; the informational Store TOAST Slack self-alert fires daily on the same condition. This tool never reclaims anything by itself. Hypertable, aggregate, table and catch-all objects carry the four TOAST fields as null. CHECKPOINTER (V137): the checkpointer block is the store's OWN checkpointer over the last self-metrics interval: write_ms and sync_ms (milliseconds in its write and sync/fsync phases), requested (checkpoints started by WAL volume reaching max_wal_size, a base backup, or a CHECKPOINT statement rather than by checkpoint_timeout; the counter does not say which), interval_seconds (MEASURED between the two sweeps that bound it, not the assumed cadence), and the cumulative counters beside them. status says whether there IS an interval: Observed, NoPrevious (one row so far), Reset (a counter went backwards — a stats reset or restart between sweeps, so no delta is stated rather than a negative one), Restarted (the store's postmaster restarted between the two sweeps, postmaster_restarted true: PostgreSQL counts the shutdown checkpoint as requested and keeps the count across the restart, and that checkpoint's own write and sync phases land in the same counters, so no delta is stated and the self-alert judges neither arm; the next sweep's interval is judged normally), or Absent; postmaster_start_time is when the store's postmaster that produced the newest row started (null on a row older than V139). The informational Store Checkpointer Pressure self-alert fires when sync_ms exceeds 10,000 or requested exceeds 0 in an interval, naming WAL sizing and refresh slicing as the levers. The checkpointer row is never an object row: it carries no bytes.")] public static async Task GetStoreMetrics( NpgsqlDataSource postgres, [Description("Days of history the window deltas and series cover. Default 30; max 400 (the series' own retention).")] int days_back = 30, @@ -765,7 +765,8 @@ internal static string CheckpointerNote(DarlingStoreMetricsReader.CheckpointerRe + $"{(syncMs / 1000.0).ToString("0.0", CultureInfo.InvariantCulture)}s total in its sync (fsync) phase and " + $"{((reading.WriteMs ?? 0) / 1000.0).ToString("0.0", CultureInfo.InvariantCulture)}s in its write phase — " + (averageSyncMs is double average ? $"an average of {(average / 1000.0).ToString("0.0", CultureInfo.InvariantCulture)}s of sync per checkpoint — " : string.Empty) - + $"and {requested} checkpoint(s) were REQUESTED (WAL-forced by max_wal_size) rather than timed."; + + $"and {requested} checkpoint(s) were REQUESTED rather than timed (started by WAL volume reaching max_wal_size, " + + "a base backup, or a CHECKPOINT statement, not by checkpoint_timeout)."; if (!reading.IsPressure) { @@ -778,8 +779,10 @@ internal static string CheckpointerNote(DarlingStoreMetricsReader.CheckpointerRe ? $"a per-checkpoint sync average past {barSeconds}s (the MCP host's own read deadline) is an I/O stall every " + "reader on the store shares — on one production store, three otherwise-unexplained read kills in a day all " + "sat inside 25.2 s and 14.0 s sync phases. " - : "a requested checkpoint means the store wrote more WAL between checkpoints than max_wal_size allows, so the " - + "checkpointer ran early and the next one is closer. ") + : "a requested checkpoint comes from WAL volume reaching max_wal_size, a base backup, or a CHECKPOINT statement, " + + "and PostgreSQL's counters do not say which; the log_checkpoints lines do. Raising max_wal_size is the remedy " + + "only for the first cause: when WAL volume did it, the store wrote more WAL between checkpoints than " + + "max_wal_size allows, so the checkpointer ran early and the next one is closer. ") + "The informational Store Checkpointer Pressure self-alert says the same."; }