Lite: refuse a newer data file, re-apply newer columns on every start, and run each due collector once after a sleep (#4727, #4728)

Darling/Darling.Tests/QsCaptureModeRouteKnobToastRungTests.cs

@@ -391,9 +391,20 @@ being the ceiling. */
         Assert.True(int.Parse(versionMatch.Groups[1].Value) >= 64);
         var start = initializer.IndexOf("if (fromVersion < 64)", StringComparison.Ordinal);
         Assert.True(start >= 0, "DuckDbInitializer has no v64 block");
-        var block = initializer[start..];
-        Assert.Contains("(\"query_store_health\", \"query_capture_mode\", \"VARCHAR\")", block, StringComparison.Ordinal);
-        Assert.Contains("(\"query_store_health\", \"wait_stats_capture_mode\", \"VARCHAR\")", block, StringComparison.Ordinal);
+        /* #4727: the two columns are entries of ONE versioned list, DuckDbInitializer.AddedColumns (version first),
+           and the v64 step adds its own version's entries through the shared AddMissingColumnsAsync. The step is cut
+           at the next "if (fromVersion <" so a call in a later step cannot satisfy the pin. */
+        var end = initializer.IndexOf("if (fromVersion <", start + 1, StringComparison.Ordinal);
+        Assert.True(end > start, "DuckDbInitializer's v64 block has no later step to end at");
+        var block = initializer[start..end];
+        var listStart = initializer.IndexOf("internal static readonly (int Version, string Table, string Column, string Type)[] AddedColumns =", StringComparison.Ordinal);
+        Assert.True(listStart >= 0, "DuckDbInitializer no longer declares AddedColumns in the pinned shape");
+        var listEnd = initializer.IndexOf("};", listStart, StringComparison.Ordinal);
+        Assert.True(listEnd > listStart, "DuckDbInitializer's AddedColumns list has no closing brace to end at");
+        var list = initializer[listStart..listEnd];
+        Assert.Contains("(64, \"query_store_health\", \"query_capture_mode\", \"VARCHAR\")", list, StringComparison.Ordinal);
+        Assert.Contains("(64, \"query_store_health\", \"wait_stats_capture_mode\", \"VARCHAR\")", list, StringComparison.Ordinal);
+        Assert.Contains("AddMissingColumnsAsync(connection, AddedColumnsForVersion(64))", block, StringComparison.Ordinal);
         Assert.Contains("Running migration to v64", block, StringComparison.Ordinal);
         Assert.Contains("twinning Darling's V137", block, StringComparison.Ordinal);
 

Darling/Darling.Tests/TimeHonestyRungTests.cs

@@ -205,9 +205,23 @@ public void TheLiteTwinIsSchemaV63_AndCarriesBothColumns()
         var declaration = System.Text.RegularExpressions.Regex.Match(lite, @"internal const int CurrentSchemaVersion = (\d+);");
         Assert.True(declaration.Success, "DuckDbInitializer no longer declares CurrentSchemaVersion in the pinned shape");
         Assert.True(int.Parse(declaration.Groups[1].Value, CultureInfo.InvariantCulture) >= 63, "Lite's schema version fell below the v63 twin");
-        var block = lite[lite.IndexOf("if (fromVersion < 63)", StringComparison.Ordinal)..];
-        Assert.Contains("(\"cpu_utilization_stats\", \"sample_time_utc\", \"TIMESTAMP\")", block, StringComparison.Ordinal);
-        Assert.Contains("(\"server_properties\", \"time_zone_id\", \"VARCHAR\")", block, StringComparison.Ordinal);
+        /* #4727: Lite's added columns are entries of ONE versioned list, DuckDbInitializer.AddedColumns (version
+           first), and each migration step adds its own version's entries through the shared AddMissingColumnsAsync.
+           So the two v63 columns are pinned as entries of that list, and the v63 step, cut at the next
+           "if (fromVersion <" so a call in a later step cannot satisfy the pin, as the one that adds version 63's. */
+        var listStart = lite.IndexOf("internal static readonly (int Version, string Table, string Column, string Type)[] AddedColumns =", StringComparison.Ordinal);
+        Assert.True(listStart >= 0, "DuckDbInitializer no longer declares AddedColumns in the pinned shape");
+        var listEnd = lite.IndexOf("};", listStart, StringComparison.Ordinal);
+        Assert.True(listEnd > listStart, "DuckDbInitializer's AddedColumns list has no closing brace to end at");
+        var list = lite[listStart..listEnd];
+        Assert.Contains("(63, \"cpu_utilization_stats\", \"sample_time_utc\", \"TIMESTAMP\")", list, StringComparison.Ordinal);
+        Assert.Contains("(63, \"server_properties\", \"time_zone_id\", \"VARCHAR\")", list, StringComparison.Ordinal);
+        var start = lite.IndexOf("if (fromVersion < 63)", StringComparison.Ordinal);
+        Assert.True(start >= 0, "DuckDbInitializer has no v63 block");
+        var end = lite.IndexOf("if (fromVersion <", start + 1, StringComparison.Ordinal);
+        Assert.True(end > start, "DuckDbInitializer's v63 block has no later step to end at");
+        var block = lite[start..end];
+        Assert.Contains("AddMissingColumnsAsync(connection, AddedColumnsForVersion(63))", block, StringComparison.Ordinal);
         Assert.Contains("twinning Darling's V134", block, StringComparison.Ordinal);
     }
 

Lite.Tests/CollectionCycleResumeTests.cs

@@ -0,0 +1,129 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using PerformanceMonitorLite.Models;
+using PerformanceMonitorLite.Services;
+using Xunit;
+
+namespace PerformanceMonitorLite.Tests;
+
+/// <summary>
+/// #4728: after the computer sleeps and resumes, the collection loop's delay returns long after the slot it waited
+/// for. Handing the collectors that stale slot ran every collector that was due then, and ran it again at the next
+/// slot a minute later. The wait returns the latest grid slot at or before now instead, so each due collector runs
+/// once. The simulation drives the production wait (<see cref="CollectionBackgroundService.WaitForNextCycleAsync"/>)
+/// with a fake clock and a fake delay, and <see cref="ScheduleManager"/>'s mark-and-due for the collectors, the way
+/// <c>RemoteCollectorService</c> marks a run with the cycle's slot.
+/// </summary>
+public sealed class CollectionCycleResumeTests : IDisposable
+{
+    private static readonly DateTime T0 = new(2026, 9, 29, 10, 0, 0, DateTimeKind.Utc);
+    private static readonly TimeSpan Interval = TimeSpan.FromMinutes(1);
+    private const string ServerId = "resume-server";
+    private const string EveryMinute = "every_minute";
+    private const string EveryFive = "every_five";
+    private const string EveryFifteen = "every_fifteen";
+
+    private readonly string _configDir;
+
+    public CollectionCycleResumeTests()
+    {
+        _configDir = Path.Combine(Path.GetTempPath(), "CollectionCycleResumeTests_" + Guid.NewGuid().ToString("N")[..8]);
+        Directory.CreateDirectory(_configDir);
+    }
+
+    public void Dispose()
+    {
+        try { if (Directory.Exists(_configDir)) Directory.Delete(_configDir, recursive: true); }
+        catch { /* best-effort cleanup */ }
+    }
+
+    private sealed record Run(string Collector, DateTime Slot, DateTime WallClock);
+
+    /// <summary>
+    /// Runs <paramref name="iterations"/> loop passes from slot T0. The delay after pass <paramref name="sleepAfterPass"/>
+    /// comes back <paramref name="sleep"/> late, the way a resume from sleep does (-1 for no sleep).
+    /// </summary>
+    private async Task<List<Run>> SimulateAsync(int iterations, int sleepAfterPass, TimeSpan sleep)
+    {
+        var schedule = new ScheduleManager(_configDir);
+        schedule.SetScheduleForServer(ServerId, new List<CollectorSchedule>
+        {
+            new() { Name = EveryMinute, Enabled = true, FrequencyMinutes = 1 },
+            new() { Name = EveryFive, Enabled = true, FrequencyMinutes = 5 },
+            new() { Name = EveryFifteen, Enabled = true, FrequencyMinutes = 15 },
+        });
+
+        var now = T0;
+        var cycleStart = T0;
+        var runs = new List<Run>();
+        for (var pass = 0; pass < iterations; pass++)
+        {
+            foreach (var due in schedule.GetDueCollectorsForServer(ServerId, cycleStart))
+            {
+                runs.Add(new Run(due.Name, cycleStart, now));
+                schedule.MarkCollectorRunForServer(ServerId, due.Name, cycleStart);
+            }
+
+            var thisPass = pass;
+            cycleStart = await CollectionBackgroundService.WaitForNextCycleAsync(
+                cycleStart,
+                Interval,
+                () => now,
+                (wait, _) =>
+                {
+                    now += wait;
+                    if (thisPass == sleepAfterPass)
+                        now += sleep;
+                    return Task.CompletedTask;
+                },
+                CancellationToken.None);
+        }
+
+        return runs;
+    }
+
+    /// <summary>
+    /// A 47-minute sleep begins while the loop waits for the slot at which the 5-minute collector becomes due. Each
+    /// due collector runs once after the resume: the 5-minute one is not run again a minute later (the old shape ran
+    /// it at the stale slot and again at the next one, 60 s apart), and the resume slot is on the grid.
+    /// </summary>
+    [Fact]
+    public async Task AfterASleepAndResume_EachDueCollectorRunsOnce_NotTwiceAMinuteApart()
+    {
+        var sleep = TimeSpan.FromMinutes(47);
+        var runs = await SimulateAsync(iterations: 12, sleepAfterPass: 4, sleep);
+
+        var resumeWall = T0 + TimeSpan.FromMinutes(5) + sleep;
+        var afterResume = runs.Where(r => r.WallClock >= resumeWall).ToList();
+
+        /* The slot handed to the collectors is the latest grid slot at or before now, not the one from before the sleep. */
+        Assert.All(afterResume, r => Assert.Equal(r.WallClock, r.Slot));
+
+        var five = afterResume.Where(r => r.Collector == EveryFive).ToList();
+        Assert.Equal(new[] { resumeWall, resumeWall + TimeSpan.FromMinutes(5) }, five.Select(r => r.WallClock).ToArray());
+
+        var fifteen = afterResume.Where(r => r.Collector == EveryFifteen).ToList();
+        Assert.Equal(new[] { resumeWall }, fifteen.Select(r => r.WallClock).ToArray());
+
+        /* A 1-minute collector still runs at the resume slot and at the next one. */
+        var minute = afterResume.Where(r => r.Collector == EveryMinute).Select(r => r.Slot).ToList();
+        Assert.Contains(resumeWall, minute);
+        Assert.Contains(resumeWall + Interval, minute);
+    }
+
+    /// <summary>Without a sleep the grid is unchanged: each collector runs at its own cadence, at exact slots.</summary>
+    [Fact]
+    public async Task WithoutASleep_EachCollectorRunsAtItsOwnCadenceOnTheGrid()
+    {
+        var runs = await SimulateAsync(iterations: 16, sleepAfterPass: -1, TimeSpan.Zero);
+
+        Assert.Equal(16, runs.Count(r => r.Collector == EveryMinute));
+        Assert.Equal(new[] { 0, 5, 10, 15 }, runs.Where(r => r.Collector == EveryFive).Select(r => (int)(r.Slot - T0).TotalMinutes).ToArray());
+        Assert.Equal(new[] { 0, 15 }, runs.Where(r => r.Collector == EveryFifteen).Select(r => (int)(r.Slot - T0).TotalMinutes).ToArray());
+        Assert.All(runs, r => Assert.Equal(r.WallClock, r.Slot));
+    }
+}

Lite.Tests/DeltaFamilyIntervalColumnTests.cs

@@ -184,12 +184,16 @@ public void TheLiteMigration_AddsTheColumnToAllFour_AtSchemaVersion60()
 
         var source = RepoSource.Read("Lite", "Database", "DuckDbInitializer.cs");
         Assert.Contains("if (fromVersion < 60)", source, StringComparison.Ordinal);
-        Assert.Contains(
-            "foreach (var table in new[] { \"wait_stats\", \"file_io_stats\", \"latch_stats\", \"spinlock_stats\" })",
-            source, StringComparison.Ordinal);
-        Assert.Contains(
-            "$\"ALTER TABLE {table} ADD COLUMN IF NOT EXISTS sample_interval_seconds INTEGER\"",
-            source, StringComparison.Ordinal);
+
+        /* #4727: the entries moved out of the step into DuckDbInitializer.AddedColumns; the step adds its own
+           version's entries through the shared AddMissingColumnsAsync, one idempotent ADD COLUMN per entry. */
+        Assert.Contains("AddMissingColumnsAsync(connection, AddedColumnsForVersion(60))", source, StringComparison.Ordinal);
+        foreach (var table in new[] { "wait_stats", "file_io_stats", "latch_stats", "spinlock_stats" })
+        {
+            Assert.Contains($"(60, \"{table}\", \"sample_interval_seconds\", \"INTEGER\")", source, StringComparison.Ordinal);
+        }
+
+        Assert.Contains("ADD COLUMN IF NOT EXISTS {column} {type}", source, StringComparison.Ordinal);
     }
 
     /// <summary>
@@ -207,18 +211,21 @@ public void TheLiteMigration_CompletesTheIntervalAndStoresTheOffsets_AtSchemaVer
         var source = RepoSource.Read("Lite", "Database", "DuckDbInitializer.cs");
         var block = source[source.IndexOf("if (fromVersion < 61)", StringComparison.Ordinal)..];
 
-        foreach (var pair in new[]
+        /* #4727: the four entries live in DuckDbInitializer.AddedColumns (INTEGER each); the step adds its own
+           version's entries through the shared AddMissingColumnsAsync. */
+        foreach (var entry in new[]
         {
-            "(\"procedure_stats\", \"sample_interval_seconds\")",
-            "(\"memory_grant_stats\", \"sample_interval_seconds\")",
-            "(\"query_stats\", \"statement_start_offset\")",
-            "(\"query_stats\", \"statement_end_offset\")",
+            "(61, \"procedure_stats\", \"sample_interval_seconds\", \"INTEGER\")",
+            "(61, \"memory_grant_stats\", \"sample_interval_seconds\", \"INTEGER\")",
+            "(61, \"query_stats\", \"statement_start_offset\", \"INTEGER\")",
+            "(61, \"query_stats\", \"statement_end_offset\", \"INTEGER\")",
         })
         {
-            Assert.Contains(pair, block, StringComparison.Ordinal);
+            Assert.Contains(entry, block, StringComparison.Ordinal);
         }
 
-        Assert.Contains("$\"ALTER TABLE {table} ADD COLUMN IF NOT EXISTS {column} INTEGER\"", block, StringComparison.Ordinal);
+        Assert.Contains("AddMissingColumnsAsync(connection, AddedColumnsForVersion(61))", block, StringComparison.Ordinal);
+        Assert.Contains("ADD COLUMN IF NOT EXISTS {column} {type}", block, StringComparison.Ordinal);
 
         /* The offsets' semantics, in the block, in these words: the pair a future reader second-guesses. */
         Assert.Contains("BYTES", block, StringComparison.Ordinal);

Lite.Tests/NewerSchemaFileGuardTests.cs

@@ -0,0 +1,97 @@
+using System;
+using System.IO;
+using System.Security.Cryptography;
+using System.Threading.Tasks;
+using DuckDB.NET.Data;
+using PerformanceMonitorLite.Database;
+using Xunit;
+
+namespace PerformanceMonitorLite.Tests;
+
+/// <summary>
+/// #4727: a data file stamped with a schema version newer than the app's is refused, with a message naming both
+/// versions, and nothing is written to it. A read-only open would not be enough: the collectors would still fail.
+/// </summary>
+public class NewerSchemaFileGuardTests : IDisposable
+{
+    private readonly string _tempDir;
+    private readonly string _dbPath;
+
+    public NewerSchemaFileGuardTests()
+    {
+        _tempDir = Path.Combine(Path.GetTempPath(), "LiteTests_" + Guid.NewGuid().ToString("N")[..8]);
+        Directory.CreateDirectory(_tempDir);
+        _dbPath = Path.Combine(_tempDir, "test.duckdb");
+    }
+
+    public void Dispose()
+    {
+        try
+        {
+            if (Directory.Exists(_tempDir))
+                Directory.Delete(_tempDir, recursive: true);
+        }
+        catch
+        {
+            /* Best-effort cleanup */
+        }
+    }
+
+    [Fact]
+    public async Task AFileStampedOneVersionAboveTheApp_IsRefused_AndNothingIsWrittenToIt()
+    {
+        var newer = DuckDbInitializer.CurrentSchemaVersion + 1;
+
+        using (var seed = new DuckDBConnection($"Data Source={_dbPath}"))
+        {
+            await seed.OpenAsync();
+            await ExecAsync(seed, "CREATE TABLE schema_version (version INTEGER NOT NULL)");
+            await ExecAsync(seed, $"INSERT INTO schema_version VALUES ({newer})");
+        }
+
+        var bytesBefore = SHA256.HashData(File.ReadAllBytes(_dbPath));
+        var writtenBefore = File.GetLastWriteTimeUtc(_dbPath);
+
+        var initializer = new DuckDbInitializer(_dbPath);
+        var refusal = await Assert.ThrowsAsync<SchemaVersionTooNewException>(() => initializer.InitializeAsync());
+
+        Assert.Equal(newer, refusal.FileVersion);
+        Assert.Equal(DuckDbInitializer.CurrentSchemaVersion, refusal.AppVersion);
+        Assert.Contains($"schema version {newer}", refusal.Message, StringComparison.Ordinal);
+        Assert.Contains($"schema version {DuckDbInitializer.CurrentSchemaVersion}", refusal.Message, StringComparison.Ordinal);
+
+        Assert.Equal(bytesBefore, SHA256.HashData(File.ReadAllBytes(_dbPath)));
+        Assert.Equal(writtenBefore, File.GetLastWriteTimeUtc(_dbPath));
+        Assert.False(File.Exists(_dbPath + ".wal"), "a refused open must not leave a write-ahead log behind");
+
+        /* And no table was created: the stamp table is still the only thing in the file. */
+        using var verify = new DuckDBConnection($"Data Source={_dbPath};ACCESS_MODE=READ_ONLY");
+        await verify.OpenAsync();
+        using var count = verify.CreateCommand();
+        count.CommandText = "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'main'";
+        Assert.Equal(1L, Convert.ToInt64(await count.ExecuteScalarAsync()));
+    }
+
+    [Fact]
+    public async Task AFileStampedAtTheAppsOwnVersion_StillOpens()
+    {
+        var first = new DuckDbInitializer(_dbPath);
+        await first.InitializeAsync();
+
+        var second = new DuckDbInitializer(_dbPath);
+        await second.InitializeAsync();
+
+        using var verify = new DuckDBConnection($"Data Source={_dbPath}");
+        await verify.OpenAsync();
+        using var version = verify.CreateCommand();
+        version.CommandText = "SELECT MAX(version) FROM schema_version";
+        Assert.Equal((long)DuckDbInitializer.CurrentSchemaVersion, Convert.ToInt64(await version.ExecuteScalarAsync()));
+    }
+
+    private static async Task ExecAsync(DuckDBConnection connection, string sql)
+    {
+        using var command = connection.CreateCommand();
+        command.CommandText = sql;
+        await command.ExecuteNonQueryAsync();
+    }
+}

Lite.Tests/PerfmonCounterTypeTests.cs

@@ -128,7 +128,11 @@ public void TheLiteMigration_StoresTheCounterType_AtSchemaVersion62()
         Assert.True(start >= 0, "DuckDbInitializer has no v62 block");
         var block = source[start..];
 
-        Assert.Contains("\"ALTER TABLE perfmon_stats ADD COLUMN IF NOT EXISTS cntr_type INTEGER\"", block, StringComparison.Ordinal);
+        /* #4727: the entry lives in DuckDbInitializer.AddedColumns; the step adds it through the shared
+           AddMissingColumnsAsync, one idempotent ADD COLUMN IF NOT EXISTS. */
+        Assert.Contains("(62, \"perfmon_stats\", \"cntr_type\", \"INTEGER\")", block, StringComparison.Ordinal);
+        Assert.Contains("AddMissingColumnsAsync(connection, AddedColumnsForVersion(62))", block, StringComparison.Ordinal);
+        Assert.Contains("ADD COLUMN IF NOT EXISTS {column} {type}", block, StringComparison.Ordinal);
         Assert.Contains("Running migration to v62", block, StringComparison.Ordinal);
         foreach (var phrase in new[] { "COUNTS", "LEVELS", "falling level", "counter reset", "NULL delta and NULL interval", "name-suffix proxy", "Appended at the end", "Nothing to backfill", "REQUIRED on this side" })
         {