Count offline CPUs and NUMA nodes correctly in the CPU scheduler collector (#5656)

.github/workflows/build.yml

@@ -647,6 +647,10 @@ jobs:
               # #5616: the Lite shard script and its packer are test inputs (Lite.Tests CrossAppGuardCiGateTests parses the script), so an edit to either has to reach the Lite suite.
               - '.github/scripts/run-lite-shard.ps1'
               - '.github/scripts/lite-shard-pack.py'
+              # #5656: Lite.Tests pins read install/ scripts as source (CpuSchedulerStatsCollectorDefinitionTests reads install/17,
+              # DmvResourceDatabaseSentinelTests reads install/56). A PR that edits only an install script fires `installer`
+              # and `sql-validation`, neither of which runs Lite.Tests, so the nightly would be the first to catch drift.
+              - 'install/**/!(*.md)'
               # #2489, changed by Erik's 2026-09-24 ruling that docs-only changes skip CI:
               # README.md and Lite/README.md used to sit here so LiteRuntimePrerequisiteDocsTests
               # (which reads these two READMEs against the built runtimeconfig and this file's
@@ -2650,6 +2654,10 @@ jobs:
               # #5616: the Lite shard script and its packer are test inputs (Lite.Tests CrossAppGuardCiGateTests parses the script), so an edit to either has to reach the Lite suite.
               - '.github/scripts/run-lite-shard.ps1'
               - '.github/scripts/lite-shard-pack.py'
+              # #5656: Lite.Tests pins read install/ scripts as source (CpuSchedulerStatsCollectorDefinitionTests reads install/17,
+              # DmvResourceDatabaseSentinelTests reads install/56). A PR that edits only an install script fires `installer`
+              # and `sql-validation`, neither of which runs Lite.Tests, so the nightly would be the first to catch drift.
+              - 'install/**/!(*.md)'
             # The rest of the build job's `lite` set: the three Darling trees Lite.Tests reads as source.
             # A diff that reaches ONLY these (no lite_shard, core_shard or root_shard entry) runs just the
             # Lite.Tests classes tagged [Trait("Reads", "Darling")], in one shard; see the scope step below.

Lite.Tests/CpuSchedulerStatsCollectorDefinitionTests.cs

@@ -7,6 +7,7 @@
  */
 
 using System;
+using System.IO;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -98,18 +99,13 @@ ELSE 0
         ),
     total_node_count = CONVERT(integer, nodes.total_nodes),
     nodes_online_count = nodes.online_nodes,
-    offline_cpu_count = nodes.offline_cpus,
+    offline_cpu_count = CONVERT(integer, offline_cpus.offline_cpu_count),
     offline_cpu_warning =
         CONVERT
         (
             bit,
             CASE
-                WHEN
-                (
-                    SELECT COUNT_BIG(*)
-                    FROM sys.dm_os_schedulers AS dos_offline
-                    WHERE dos_offline.is_online = 0
-                ) > 0
+                WHEN offline_cpus.offline_cpu_count > 0
                 THEN 1
                 ELSE 0
             END
@@ -125,24 +121,23 @@ CROSS JOIN
             SUM
             (
                 CASE
-                    WHEN n.node_state_desc = N'ONLINE'
-                    THEN 1
-                    ELSE 0
-                END
-            ),
-        offline_cpus =
-            SUM
-            (
-                CASE
-                    WHEN n.node_state_desc <> N'ONLINE'
+                    WHEN n.node_state_desc NOT LIKE N'OFFLINE%'
                     THEN 1
                     ELSE 0
                 END
             )
     FROM sys.dm_os_nodes AS n
-    WHERE n.node_id <> 32767 /*Exclude DAC node*/
+    WHERE n.node_state_desc NOT LIKE N'%DAC%' /*Exclude the DAC node by its state, not by node_id (#5656)*/
 ) AS nodes
 CROSS JOIN
+(
+    /*Offline CPUs are user schedulers SQL Server has taken offline; the warning flag reads this same count*/
+    SELECT
+        offline_cpu_count = COUNT_BIG(*)
+    FROM sys.dm_os_schedulers AS dos_offline
+    WHERE dos_offline.status = N'VISIBLE OFFLINE'
+) AS offline_cpus
+CROSS JOIN
 (
     SELECT
         active_request_count = SUM(wg.active_request_count),
@@ -198,7 +193,7 @@ GROUP BY
     osm.system_memory_state_desc,
     nodes.total_nodes,
     nodes.online_nodes,
-    nodes.offline_cpus
+    offline_cpus.offline_cpu_count
 OPTION(RECOMPILE);";
 
     [Fact]
@@ -212,6 +207,53 @@ public void Query_IsTheVerbatimParityContract()
         Assert.Equal("cpu_scheduler_stats", CpuSchedulerStatsCollector.Instance.TargetTable);
     }
 
+    /* #5656: the shared collector SQL and the Dashboard's install/17 script both counted the DAC node as
+       an offline node (it was filtered by node_id, but the DAC node's id is 64), matched "online" with an
+       exact N'ONLINE' (so ONLINE DAC, IDLE and the combined states read as offline), and reported nodes as
+       "offline CPUs" while the warning flag counted schedulers. Pin both copies so they cannot drift back. */
+    [Theory]
+    [InlineData("collector")]
+    [InlineData("install")]
+    public void NodeAndOfflineCpuCounts_UseStateRulesAndOneSchedulerPredicate(string copy)
+    {
+        var sql = copy == "collector"
+            ? CpuSchedulerStatsCollector.Instance.BuildQuery(CollectorTestContext.Make(new RecordingCollectorDeltaCalculator())).Text
+            : CollectionSelect(File.ReadAllText(FindRepoFile(Path.Combine("install", "17_collect_cpu_scheduler_stats.sql"))));
+
+        /* The DAC node is left out by its state, never by a node_id; "online" is every state except OFFLINE. */
+        Assert.Contains("n.node_state_desc NOT LIKE N'%DAC%'", sql);
+        Assert.Contains("n.node_state_desc NOT LIKE N'OFFLINE%'", sql);
+        Assert.DoesNotContain("32767", sql);
+        Assert.DoesNotContain("node_id <>", sql);
+        Assert.DoesNotContain("node_state_desc = N'ONLINE'", sql);
+        Assert.DoesNotContain("node_state_desc <> N'ONLINE'", sql);
+
+        /* Offline CPUs are counted from the user schedulers SQL Server took offline, once, and the warning
+           flag reads that same count (no second is_online = 0 subquery that can disagree with it). */
+        Assert.Equal(1, CountOccurrences(sql, "= N'VISIBLE OFFLINE'"));
+        Assert.DoesNotContain("is_online", sql);
+        Assert.Contains("offline_cpu_count = CONVERT(integer, offline_cpus.offline_cpu_count)", sql);
+        Assert.Contains("WHEN offline_cpus.offline_cpu_count > 0", sql);
+        Assert.DoesNotContain("nodes.offline_cpus", sql);
+    }
+
+    [Fact]
+    public void InstallScript_DebugMessage_NamesOfflineCpusNotNodes()
+    {
+        var sql = File.ReadAllText(FindRepoFile(Path.Combine("install", "17_collect_cpu_scheduler_stats.sql")));
+        Assert.Contains("Offline CPUs detected: %d offline CPUs", sql);
+        Assert.DoesNotContain("offline nodes", sql);
+    }
+
+    /* The collection_log row is what an operator reads back after the fact, and the count no longer counts nodes. */
+    [Fact]
+    public void InstallScript_CollectionLogText_NamesOfflineCpusNotNodes()
+    {
+        var sql = File.ReadAllText(FindRepoFile(Path.Combine("install", "17_collect_cpu_scheduler_stats.sql")));
+        Assert.Contains("N'CRITICAL: Offline CPUs detected'", sql);
+        Assert.DoesNotContain("in NUMA nodes", sql);
+    }
+
     [Fact]
     public void AppliesTo_SkipsAzureSqlDb_ButCollectsOnPremAndManagedInstance()
     {
@@ -327,4 +369,45 @@ public void WritePayload_EmitsPayloadOrder_PointInTime_NoDeltas()
         /* Point-in-time snapshot — no delta calculator interaction at all. */
         Assert.Empty(deltas.Calls);
     }
+
+    /* The install proc's collection SELECT, from its first select-list column to its OPTION(RECOMPILE). The
+       whole-file DoesNotContain checks (32767, is_online) belong to this slice, not to the rest of the proc. Both
+       anchors are asserted, so a rename cannot leave an empty slice that passes every DoesNotContain. */
+    private static string CollectionSelect(string installScript)
+    {
+        const string startAnchor = "max_workers_count = osi.max_workers_count";
+        const string endAnchor = "OPTION(RECOMPILE);";
+        var start = installScript.IndexOf(startAnchor, StringComparison.Ordinal);
+        Assert.True(start >= 0, $"install/17 no longer contains the collection SELECT start anchor: {startAnchor}");
+        var end = installScript.IndexOf(endAnchor, start, StringComparison.Ordinal);
+        Assert.True(end > start, $"install/17 no longer contains {endAnchor} after the collection SELECT start");
+        return installScript.Substring(start, end + endAnchor.Length - start);
+    }
+
+    private static int CountOccurrences(string text, string needle)
+    {
+        var count = 0;
+        var index = 0;
+        while ((index = text.IndexOf(needle, index, StringComparison.Ordinal)) >= 0)
+        {
+            count++;
+            index += needle.Length;
+        }
+        return count;
+    }
+
+    private static string FindRepoFile(string relativePath)
+    {
+        var dir = AppContext.BaseDirectory;
+        for (var i = 0; i < 8 && dir is not null; i++)
+        {
+            var candidate = Path.Combine(dir, relativePath);
+            if (File.Exists(candidate))
+            {
+                return candidate;
+            }
+            dir = Path.GetDirectoryName(dir);
+        }
+        throw new FileNotFoundException($"Could not locate {relativePath} walking up from {AppContext.BaseDirectory}");
+    }
 }

PerformanceMonitor.Collectors/CpuSchedulerStatsCollector.cs

@@ -151,18 +151,13 @@ ELSE 0
         ),
     total_node_count = CONVERT(integer, nodes.total_nodes),
     nodes_online_count = nodes.online_nodes,
-    offline_cpu_count = nodes.offline_cpus,
+    offline_cpu_count = CONVERT(integer, offline_cpus.offline_cpu_count),
     offline_cpu_warning =
         CONVERT
         (
             bit,
             CASE
-                WHEN
-                (
-                    SELECT COUNT_BIG(*)
-                    FROM sys.dm_os_schedulers AS dos_offline
-                    WHERE dos_offline.is_online = 0
-                ) > 0
+                WHEN offline_cpus.offline_cpu_count > 0
                 THEN 1
                 ELSE 0
             END
@@ -178,24 +173,23 @@ CROSS JOIN
             SUM
             (
                 CASE
-                    WHEN n.node_state_desc = N'ONLINE'
-                    THEN 1
-                    ELSE 0
-                END
-            ),
-        offline_cpus =
-            SUM
-            (
-                CASE
-                    WHEN n.node_state_desc <> N'ONLINE'
+                    WHEN n.node_state_desc NOT LIKE N'OFFLINE%'
                     THEN 1
                     ELSE 0
                 END
             )
     FROM sys.dm_os_nodes AS n
-    WHERE n.node_id <> 32767 /*Exclude DAC node*/
+    WHERE n.node_state_desc NOT LIKE N'%DAC%' /*Exclude the DAC node by its state, not by node_id (#5656)*/
 ) AS nodes
 CROSS JOIN
+(
+    /*Offline CPUs are user schedulers SQL Server has taken offline; the warning flag reads this same count*/
+    SELECT
+        offline_cpu_count = COUNT_BIG(*)
+    FROM sys.dm_os_schedulers AS dos_offline
+    WHERE dos_offline.status = N'VISIBLE OFFLINE'
+) AS offline_cpus
+CROSS JOIN
 (
     SELECT
         active_request_count = SUM(wg.active_request_count),
@@ -251,7 +245,7 @@ GROUP BY
     osm.system_memory_state_desc,
     nodes.total_nodes,
     nodes.online_nodes,
-    nodes.offline_cpus
+    offline_cpus.offline_cpu_count
 OPTION(RECOMPILE);";
 
     public override string Name => "cpu_scheduler_stats";

install/17_collect_cpu_scheduler_stats.sql

@@ -180,15 +180,10 @@ BEGIN
                 END,
             total_node_count = nodes.total_nodes,
             nodes_online_count = nodes.online_nodes,
-            offline_cpu_count = nodes.offline_cpus,
+            offline_cpu_count = CONVERT(integer, offline_cpus.offline_cpu_count),
             offline_cpu_warning =
                 CASE
-                    WHEN 
-                    (
-                        SELECT COUNT_BIG(*) 
-                        FROM sys.dm_os_schedulers dos 
-                        WHERE dos.is_online = 0
-                    ) > 0
+                    WHEN offline_cpus.offline_cpu_count > 0
                     THEN 1
                     ELSE 0
                 END
@@ -203,24 +198,23 @@ BEGIN
                     SUM
                     (
                         CASE
-                            WHEN n.node_state_desc = N'ONLINE'
-                            THEN 1
-                            ELSE 0
-                        END
-                    ),
-                offline_cpus =
-                    SUM
-                    (
-                        CASE
-                            WHEN n.node_state_desc <> N'ONLINE'
+                            WHEN n.node_state_desc NOT LIKE N'OFFLINE%'
                             THEN 1
                             ELSE 0
                         END
                     )
             FROM sys.dm_os_nodes AS n
-            WHERE n.node_id <> 32767 /*Exclude DAC node*/
+            WHERE n.node_state_desc NOT LIKE N'%DAC%' /*Exclude the DAC node by its state, not by node_id (#5656)*/
         ) AS nodes
         CROSS JOIN
+        (
+            /*Offline CPUs are user schedulers SQL Server has taken offline; the warning flag reads this same count*/
+            SELECT
+                offline_cpu_count = COUNT_BIG(*)
+            FROM sys.dm_os_schedulers AS dos_offline
+            WHERE dos_offline.status = N'VISIBLE OFFLINE'
+        ) AS offline_cpus
+        CROSS JOIN
         (
             SELECT
                 active_request_count =
@@ -280,7 +274,7 @@ BEGIN
             osm.system_memory_state_desc,
             nodes.total_nodes,
             nodes.online_nodes,
-            nodes.offline_cpus
+            offline_cpus.offline_cpu_count
         OPTION(RECOMPILE);
 
         SET @rows_collected = ROWCOUNT_BIG();
@@ -316,7 +310,7 @@ BEGIN
             (
                 N'cpu_scheduler_stats_collector',
                 N'CRITICAL',
-                N'CRITICAL: Offline CPUs detected in NUMA nodes'
+                N'CRITICAL: Offline CPUs detected'
             );
         END;
 
@@ -393,7 +387,7 @@ BEGIN
 
             IF @offline_cpu_warning = 1
             BEGIN
-                RAISERROR(N'CRITICAL: Offline CPUs detected: %d offline nodes', 0, 1, @current_offline_cpus) WITH NOWAIT;
+                RAISERROR(N'CRITICAL: Offline CPUs detected: %d offline CPUs', 0, 1, @current_offline_cpus) WITH NOWAIT;
             END;
         END;