From f8c9d67e52c3d42e35d3860d03b5b180c3edc7f1 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 18 Sep 2026 10:53:52 -0400 Subject: [PATCH 1/2] Extreme, corroborated anomaly findings can cross the notify floor (#3526) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every ANOMALY_* ramp saturates its base at 1.0, the Layer-3 tuning-class cap holds the final at 1.49, no amplifier arm existed, and the shipped notify floor is 1.5 — so the baseline engine could never page at default settings. FactScorer: - IsExtremeAnomaly: a per-fact escape from the tuning-class cap for ANOMALY_* only, at 3x the cutoff the fact fired at (10.5σ robust, 15σ heavy-tail, 6σ classical; 3x the absolute bar on the low-quality fallback path, never for is_new wait profiles; ratio/count/delta families stay capped). - AnomalyAmplifiers: a co-fire arm — sibling anomaly families and measured absolute facts corroborate at +0.2/+0.3, so an extreme anomaly with two corroborators scores 1.6 and a lone or routine one cannot reach 1.5. - The impact-peer escape and CXPACKET's cap are unchanged. Lite.Tests/FactScorerTests: 12 new pins on the escape bar arithmetic per path, the co-fire arm, the routine/lone/single-corroborator negatives, the low-quality sigma blindness, CXPACKET staying capped beside extreme anomalies. --- Lite.Tests/FactScorerTests.cs | 320 +++++++++++++++++++ PerformanceMonitor.Analysis/FactScorer.cs | 371 +++++++++++++++++++++- 2 files changed, 678 insertions(+), 13 deletions(-) diff --git a/Lite.Tests/FactScorerTests.cs b/Lite.Tests/FactScorerTests.cs index 2f9dec3a58..93f2dd9d10 100644 --- a/Lite.Tests/FactScorerTests.cs +++ b/Lite.Tests/FactScorerTests.cs @@ -457,6 +457,326 @@ public void Score_CxPacketWithSignificantResourceSemaphore_CapReleased() Assert.True(cx.Severity >= 1.5, "a significant RESOURCE_SEMAPHORE releases the cap into CRITICAL"); } + /* ── Layer 3: #3526 anomaly extremity escape + the anomaly co-fire amplifier arm ── */ + + // Before #3526 every ANOMALY_* fact was capped at 1.49 by the tuning-class cap, had NO amplifier arm + // (so its final severity was its <= 1.0 base), and the shipped notify floor is 1.5: the baseline + // engine could never page. These tests pin the two halves of the fix — an anomaly whose deviation is + // EXTREME (>= 3x the cutoff it fired at) leaves the cap, and the new co-fire arm can carry an + // extreme, CORROBORATED anomaly past 1.5 — and, just as deliberately, everything the fix must NOT do. + + // A deviation-scored anomaly on a TRUSTWORTHY baseline (baseline_low_quality = 0). fireThreshold 0 + // means "omit the key" — a pre-#1743 fact, which the scorer anchors at the classical 2.0. + private static Fact TrustedAnomaly(string key, double deviationSigma, double fireThreshold = 0.0) + { + var metadata = new Dictionary + { + ["deviation_sigma"] = deviationSigma, + ["baseline_low_quality"] = 0.0, + ["fallback_exceedance"] = 0.0, + ["confidence"] = 1.0 + }; + if (fireThreshold > 0) metadata["fire_threshold"] = fireThreshold; + return new Fact { Source = "anomaly", Key = key, Value = 1, Metadata = metadata }; + } + + // (a) The issue's own 3am shape: a 20σ session spike (robust 3.5 cutoff → the escape opens at 10.5σ) + // beside a 15σ batch-request anomaly and SQL CPU at 85%. Before the fix the best it could do was 1.0. + // Now: base 1.0 (saturated), extreme → cap released, two co-fires (+0.3 batch, +0.3 CPU >= 80) → 1.6. + // The batch anomaly is extreme too (15 >= 10.5) and is corroborated symmetrically (+0.3 sessions, + // +0.3 CPU) → 1.6 as well. Both clear the shipped 1.5 notify floor for the first time. + [Fact] + public void Score_ExtremeCorroboratedAnomaly_CrossesNotifyFloor() + { + var facts = new List + { + TrustedAnomaly("ANOMALY_SESSION_SPIKE", 20.0, 3.5), + TrustedAnomaly("ANOMALY_BATCH_REQUESTS", 15.0, 3.5), + new() { Source = "cpu", Key = "CPU_SQL_PERCENT", Value = 85 }, + }; + + new FactScorer().ScoreAll(facts); + + var sessions = facts.First(f => f.Key == "ANOMALY_SESSION_SPIKE"); + Assert.Equal(1.0, sessions.BaseSeverity, precision: 4); + Assert.Equal(1.6, sessions.Severity, precision: 4); + Assert.True(sessions.Severity >= 1.5, "an extreme, twice-corroborated anomaly must reach the CRITICAL band"); + Assert.Contains(sessions.AmplifierResults, a => a.Matched && a.Description.StartsWith("Batch-request anomaly co-fired")); + Assert.Contains(sessions.AmplifierResults, a => a.Matched && a.Description.StartsWith("SQL Server CPU >= 80%")); + + var batch = facts.First(f => f.Key == "ANOMALY_BATCH_REQUESTS"); + Assert.Equal(1.6, batch.Severity, precision: 4); + } + + // (b) The SAME fact set with the session spike at exactly its fire threshold: base 0.5, the two + // co-fires lift it to 0.8, and it stays far inside WARNING. Corroboration alone cannot page a + // fact that only just fired. + [Fact] + public void Score_AnomalyAtFireThreshold_WithCoFires_StaysWarning() + { + var facts = new List + { + TrustedAnomaly("ANOMALY_SESSION_SPIKE", 3.5, 3.5), // exactly at the cutoff → base 0.5 + TrustedAnomaly("ANOMALY_BATCH_REQUESTS", 15.0, 3.5), + new() { Source = "cpu", Key = "CPU_SQL_PERCENT", Value = 85 }, + }; + + new FactScorer().ScoreAll(facts); + + var sessions = facts.First(f => f.Key == "ANOMALY_SESSION_SPIKE"); + Assert.Equal(0.5, sessions.BaseSeverity, precision: 4); + Assert.Equal(0.8, sessions.Severity, precision: 4); // 0.5 x (1 + 0.3 + 0.3) + Assert.True(sessions.Severity <= 1.49, "an anomaly at its fire threshold must stay in WARNING"); + } + + // The routine-but-saturated case the cap exists for: 7σ on the 3.5 cutoff is 2x the anchor — the + // ramp's saturation point (base 1.0) but NOT extreme (the escape needs 10.5σ). Three co-fires push + // the Layer-2 product to 1.9, and the cap holds it at 1.49: a busy evening does not page. + [Fact] + public void Score_SaturatedButRoutineAnomaly_WithThreeCoFires_CappedAtWarningCeiling() + { + var facts = new List + { + TrustedAnomaly("ANOMALY_SESSION_SPIKE", 7.0, 3.5), // 2x anchor → base 1.0, not extreme + TrustedAnomaly("ANOMALY_BATCH_REQUESTS", 7.0, 3.5), // +0.3 + TrustedAnomaly("ANOMALY_CPU_SPIKE", 7.0, 3.5), // +0.3 + new() { Source = "cpu", Key = "CPU_SQL_PERCENT", Value = 85 }, // +0.3 + }; + + new FactScorer().ScoreAll(facts); + + var sessions = facts.First(f => f.Key == "ANOMALY_SESSION_SPIKE"); + Assert.Equal(1.0, sessions.BaseSeverity, precision: 4); + // 1.0 x (1 + 0.3 + 0.3 + 0.3) = 1.9 pre-cap; the anomaly is not extreme, so Layer 3 caps it. + Assert.Equal(1.49, sessions.Severity, precision: 4); + Assert.True(sessions.Severity < 1.5, "saturation is not extremity — a routine 2x-anchor anomaly must stay in WARNING"); + } + + // The escape bar is 3x the cutoff the fact FIRED at, on every path the detectors take: classical 2.0 + // (a pre-#1743 fact with no fire_threshold) → 6σ; robust 3.5 → 10.5σ; heavy-tail 5.0 → 15σ. Each + // row plants the root beside the same two co-fires (+0.3 +0.3 → 1.6 when released, 1.49 when + // capped) and probes one side of the bar. The root's base is 1.0 in every row (all are >= 2x anchor). + [Theory] + [InlineData(6.0, 0.0, 1.6)] // classical: 3 x 2.0 → released + [InlineData(5.99, 0.0, 1.49)] // classical: a hair under → capped + [InlineData(10.5, 3.5, 1.6)] // robust: 3 x 3.5 → released + [InlineData(10.49, 3.5, 1.49)] // robust: a hair under → capped + [InlineData(15.0, 5.0, 1.6)] // heavy-tail: 3 x 5.0 → released + [InlineData(14.99, 5.0, 1.49)] // heavy-tail: a hair under → capped + public void Score_AnomalyExtremityEscape_OpensAtThreeTimesTheFireThreshold( + double deviationSigma, double fireThreshold, double expected) + { + var facts = new List + { + TrustedAnomaly("ANOMALY_QUERY_DURATION", deviationSigma, fireThreshold), + TrustedAnomaly("ANOMALY_BATCH_REQUESTS", 15.0, 3.5), // +0.3 + new() { Source = "cpu", Key = "CPU_SQL_PERCENT", Value = 85 }, // +0.3 + }; + + new FactScorer().ScoreAll(facts); + + var root = facts.First(f => f.Key == "ANOMALY_QUERY_DURATION"); + Assert.Equal(1.0, root.BaseSeverity, precision: 4); + Assert.Equal(expected, root.Severity, precision: 4); + } + + // (c) The never-blind fallback path: on an untrustworthy baseline the stored deviation_sigma is a + // meaningless z the detector refused to trust, so the escape must NOT read it — a planted 30σ (far + // past any sigma bar) must not release the cap. The fallback path escapes only when the ABSOLUTE + // exceedance is equally extreme: 3x the fallback bar (for batch requests, 15,000/s against the + // 5,000/s bar). 2.5x saturates the base at 1.0 but stays capped; 3.0x is released. + [Theory] + [InlineData(2.5, 1.49)] // past saturation (2x), under the 3x escape bar → capped + [InlineData(2.99, 1.49)] // a hair under → capped + [InlineData(3.0, 1.6)] // 3x the absolute bar → released, corroborated → pages + public void Score_LowQualityAnomaly_EscapesOnlyAtThreeTimesItsAbsoluteBar_NeverOnSigma( + double exceedance, double expected) + { + var facts = new List + { + new() + { + Source = "anomaly", + Key = "ANOMALY_BATCH_REQUESTS", + Value = 1, + Metadata = new() + { + ["deviation_sigma"] = 30.0, // would be extreme on the trusted path; must be ignored here + ["fire_threshold"] = 3.5, + ["baseline_low_quality"] = 1.0, + ["fallback_exceedance"] = exceedance, + ["confidence"] = 1.0 + } + }, + TrustedAnomaly("ANOMALY_SESSION_SPIKE", 15.0, 3.5), // +0.3 + new() { Source = "cpu", Key = "CPU_SQL_PERCENT", Value = 85 }, // +0.3 + }; + + new FactScorer().ScoreAll(facts); + + var batch = facts.First(f => f.Key == "ANOMALY_BATCH_REQUESTS"); + Assert.Equal(1.0, batch.BaseSeverity, precision: 4); // the fallback ramp saturates at 2x the bar + Assert.Equal(expected, batch.Severity, precision: 4); + } + + // (d) The cap's original target is untouched: CXPACKET with its non-impact amplifiers (1.7 pre-cap) + // stays at 1.49 even when two EXTREME anomalies co-fire beside it — anomalies are not impact peers, + // and the extremity escape is per-fact, for ANOMALY_* only. In the same fact set the anomalies + // themselves ARE released (each is extreme and corroborated by the other + CPU): the two escapes + // are independent. + [Fact] + public void Score_CxPacketWithExtremeAnomalyCoFires_StillCapped_AnomaliesReleased() + { + var facts = new List + { + new() { Source = "waits", Key = "CXPACKET", Value = 0.80 }, // base 1.0 + new() { Source = "config", Key = "CONFIG_CTFP", Value = 5 }, // +0.3 + new() { Source = "config", Key = "CONFIG_MAXDOP", Value = 0 }, // +0.2 + new() { Source = "queries", Key = "QUERY_HIGH_DOP", Value = 10 }, // +0.2 + TrustedAnomaly("ANOMALY_SESSION_SPIKE", 20.0, 3.5), + TrustedAnomaly("ANOMALY_BATCH_REQUESTS", 15.0, 3.5), + new() { Source = "cpu", Key = "CPU_SQL_PERCENT", Value = 85 }, + }; + + new FactScorer().ScoreAll(facts); + + var cx = facts.First(f => f.Key == "CXPACKET"); + Assert.Equal(1.49, cx.Severity, precision: 4); + Assert.True(cx.Severity < 1.5, "extreme anomalies are not impact peers — CXPACKET must stay capped"); + + Assert.Equal(1.6, facts.First(f => f.Key == "ANOMALY_SESSION_SPIKE").Severity, precision: 4); + Assert.Equal(1.6, facts.First(f => f.Key == "ANOMALY_BATCH_REQUESTS").Severity, precision: 4); + } + + // (e) A LONE extreme anomaly does not page. 25σ is the detectors' display cap — the most extreme + // reading a fact can carry — and with nothing else moving it scores exactly its 1.0 base: the cap + // is released but there is nothing for the release to preserve. The CRITICAL band is earned only + // with corroboration (the rule every base fact follows), and a solitary extreme reading is the shape + // a collector hiccup or a variance-collapsed baseline pinned at the display cap produces. + [Fact] + public void Score_LoneExtremeAnomaly_StaysAtBase_DoesNotPage() + { + var facts = new List { TrustedAnomaly("ANOMALY_SESSION_SPIKE", 25.0, 3.5) }; + + new FactScorer().ScoreAll(facts); + + var sessions = facts[0]; + Assert.Equal(1.0, sessions.BaseSeverity, precision: 4); + Assert.Equal(1.0, sessions.Severity, precision: 4); + Assert.True(sessions.Severity < 1.5, "a single uncorroborated anomaly must not page, however extreme"); + Assert.All(sessions.AmplifierResults, a => Assert.False(a.Matched)); + } + + // One corroborator is not enough either: extreme + a single +0.3 co-fire = 1.3, WARNING. Two + // independent corroborators is the bar (see the AnomalyAmplifiers worked numbers). + [Fact] + public void Score_ExtremeAnomaly_SingleCoFire_StaysWarning() + { + var facts = new List + { + TrustedAnomaly("ANOMALY_SESSION_SPIKE", 20.0, 3.5), + TrustedAnomaly("ANOMALY_BATCH_REQUESTS", 15.0, 3.5), // the only co-fire: +0.3 + }; + + new FactScorer().ScoreAll(facts); + + var sessions = facts.First(f => f.Key == "ANOMALY_SESSION_SPIKE"); + Assert.Equal(1.3, sessions.Severity, precision: 4); + Assert.True(sessions.Severity < 1.5, "one corroborator lifts an extreme anomaly, but not into CRITICAL"); + } + + // A family never corroborates itself: the CPU anomaly's arm carries the other three load siblings and + // the measured-CPU confirmation, but no "CPU anomaly co-fired" entry. + [Fact] + public void Score_AnomalyCoFireArm_DoesNotSelfCorroborate() + { + var facts = new List { TrustedAnomaly("ANOMALY_CPU_SPIKE", 12.0, 3.5) }; + + new FactScorer().ScoreAll(facts); + + var cpu = facts[0]; + Assert.DoesNotContain(cpu.AmplifierResults, a => a.Description.StartsWith("CPU anomaly co-fired")); + Assert.Contains(cpu.AmplifierResults, a => a.Description.StartsWith("Session-count anomaly co-fired")); + Assert.Contains(cpu.AmplifierResults, a => a.Description.StartsWith("Batch-request anomaly co-fired")); + Assert.Contains(cpu.AmplifierResults, a => a.Description.StartsWith("Query-duration anomaly co-fired")); + Assert.Contains(cpu.AmplifierResults, a => a.Description.StartsWith("SQL Server CPU >= 80%")); + } + + // The wait-profile family: graded and escaped off the SAME statistic. On the robust trigger the + // escape opens at 3 x the 5.0 heavy-tail cutoff = 15σ (also the ramp's saturation point); read- + // latency and query-duration anomalies corroborate at +0.3 each → 1.6. A hair under 15σ stays + // capped at 1.49 with the same co-fires. + [Theory] + [InlineData(15.0, 1.6)] + [InlineData(14.99, 1.49)] + public void Score_WaitProfileAnomaly_EscapesAtThreeTimesTheHeavyTailCutoff(double modifiedZ, double expected) + { + var facts = new List + { + new() + { + Source = "anomaly", + Key = "ANOMALY_WAIT_PROFILE", + Value = 1, + Metadata = new() { ["modified_z"] = modifiedZ, ["ratio"] = 3.0, ["is_new"] = 0 } + }, + TrustedAnomaly("ANOMALY_READ_LATENCY", 12.0, 3.5), // +0.3 + TrustedAnomaly("ANOMALY_QUERY_DURATION", 15.0, 5.0), // +0.3 + }; + + new FactScorer().ScoreAll(facts); + + var profile = facts.First(f => f.Key == "ANOMALY_WAIT_PROFILE"); + Assert.Equal(expected, profile.Severity, precision: 4); + } + + // A first-occurrence wait profile (is_new) carries the NoBaselineRatio sentinel (100 — far past any + // ratio bar) as a scoring device, not a measurement. It must never escape: "no baseline" cannot be + // "extreme against baseline". With the same two co-fires the 1.6 product is capped to 1.49. + [Fact] + public void Score_IsNewWaitProfile_NeverEscapes_DespiteSentinelRatio() + { + var facts = new List + { + new() + { + Source = "anomaly", + Key = "ANOMALY_WAIT_PROFILE", + Value = 1, + Metadata = new() { ["modified_z"] = 0.0, ["ratio"] = PerformanceMonitor.Analysis.Baselines.AnomalyThresholds.NoBaselineRatio, ["is_new"] = 1 } + }, + TrustedAnomaly("ANOMALY_READ_LATENCY", 12.0, 3.5), // +0.3 + TrustedAnomaly("ANOMALY_QUERY_DURATION", 15.0, 5.0), // +0.3 + }; + + new FactScorer().ScoreAll(facts); + + var profile = facts.First(f => f.Key == "ANOMALY_WAIT_PROFILE"); + Assert.Equal(1.0, profile.BaseSeverity, precision: 4); // the sentinel ratio saturates the ramp + Assert.Equal(1.49, profile.Severity, precision: 4); + } + + // The ratio/count families are NOT released: a blocking spike at 50x its baseline rate (base 1.0, + // saturated) has no extremity arm and no co-fire arm — it scores its base, and the cap is moot. Its + // impact reaches CRITICAL through the never-capped BLOCKING_EVENTS / DEADLOCKS keys instead. + [Fact] + public void Score_BlockingSpikeAnomaly_NoEscape_NoArm() + { + var facts = new List + { + new() { Source = "anomaly", Key = "ANOMALY_BLOCKING_SPIKE", Value = 1, Metadata = new() { ["ratio"] = 50.0, ["is_new"] = 0 } }, + TrustedAnomaly("ANOMALY_SESSION_SPIKE", 20.0, 3.5), + new() { Source = "cpu", Key = "CPU_SQL_PERCENT", Value = 85 }, + }; + + new FactScorer().ScoreAll(facts); + + var spike = facts.First(f => f.Key == "ANOMALY_BLOCKING_SPIKE"); + Assert.Equal(1.0, spike.BaseSeverity, precision: 4); + Assert.Equal(1.0, spike.Severity, precision: 4); + Assert.Empty(spike.AmplifierResults); + } + // THREADPOOL is the impact-bearing escalation path and is NEVER capped. The raised CXPACKET // amplifier (+0.5) and the new runnable-queue amplifier (+0.5, off the RUNNABLE_TASKS context fact) // drive a genuine parallelism -> thread-exhaustion meltdown to CRITICAL through THREADPOOL. diff --git a/PerformanceMonitor.Analysis/FactScorer.cs b/PerformanceMonitor.Analysis/FactScorer.cs index 2995fd9ac7..aa4f6f12e9 100644 --- a/PerformanceMonitor.Analysis/FactScorer.cs +++ b/PerformanceMonitor.Analysis/FactScorer.cs @@ -10,6 +10,8 @@ namespace PerformanceMonitor.Analysis; /// /// Layer 1: Base severity 0.0-1.0 from thresholds alone. /// Layer 2: Amplifiers multiply base up to 2.0 max using corroborating facts. +/// Layer 3: Tuning-class keys (parallelism, high-DOP queries, the routine ANOMALY_* case) are capped at +/// the WARNING ceiling unless an impact peer co-fired or, for an anomaly, its own deviation is extreme. /// /// Formula: severity = min(base * (1.0 + sum(amplifiers)), 2.0) /// @@ -104,11 +106,26 @@ public void ScoreAll(List facts) || HasSignificantWait(factsByKey, "SOS_SCHEDULER_YIELD", 0.25) || HasSignificantWait(factsByKey, "RESOURCE_SEMAPHORE", 0.10); + // #3526: the SECOND escape, per-fact and for ANOMALY_* only — extremity. The impact-peer + // escape above is the right shape for CXPACKET (parallelism is only an outage when a thread/CPU/ + // grant peer says so), but applied to every ANOMALY_* it made the baseline engine notification- + // inert at shipped settings: every anomaly ramp saturates its BASE at 1.0, the cap holds the FINAL + // at 1.49, and the notify floor is 1.5 — so a 20σ session spike beside a 15σ batch-request spike + // at 3am produced nothing that could page unless THREADPOOL/SOS/RESOURCE_SEMAPHORE happened to be + // in the picture too. The product's best per-server-calibrated statistics were structurally its + // quietest. An anomaly whose deviation is EXTREME against the cutoff it fired at (IsExtremeAnomaly: + // 3x its own fire threshold — 10.5σ on the 3.5σ robust path, 15σ on the 5.0σ heavy-tail path, 6σ + // classical; 3x the absolute bar on the never-blind fallback path) is released from the cap on its + // own evidence. Releasing the cap does NOT page by itself: base still maxes at 1.0, so the CRITICAL + // band is reached only through the anomaly co-fire amplifiers (AnomalyAmplifiers) — corroboration + // stays the house rule for >= 1.5, the escape merely stops the cap from discarding it. The + // impact-peer escape is unchanged and still releases everything; CXPACKET / CXCONSUMER / + // QUERY_HIGH_DOP have no extremity arm and stay capped however many anomalies co-fire beside them. if (!impactPeerCoFired) { foreach (var fact in facts) { - if (IsTuningClassKey(fact.Key)) + if (IsTuningClassKey(fact.Key) && !IsExtremeAnomaly(fact)) fact.Severity = Math.Min(fact.Severity, TuningClassSeverityCeiling); } } @@ -596,19 +613,112 @@ private static double ScoreBadActorFact(Fact fact) // Layer-3 tuning-class severity ceiling (see ScoreAll). Parallelism/anomaly signals describe a tuning // opportunity, not an outage — their FINAL severity is capped here (bands are >= 1.5 CRITICAL) unless an - // impact peer co-fired. 1.49 keeps a capped fact in the WARNING band without touching SeverityBand. + // impact peer co-fired or (#3526, ANOMALY_* only) the anomaly's own deviation is extreme. 1.49 keeps a + // capped fact in the WARNING band without touching SeverityBand. private const double TuningClassSeverityCeiling = 1.49; + /* #3526: the extremity escape's multiple (see IsExtremeAnomaly). An anomaly leaves the tuning-class cap + when its deviation is this many times the cutoff it FIRED at. Every deviation ramp saturates its base + at 2x its anchor (ScoreAnomalyFact: 0.5 at the anchor, 1.0 at 2x), so 3x sits a full anchor PAST the + point where the ramp stopped distinguishing — "extreme" means "so far out the scorer ran out of + scale", not "the top of the ramp". The arithmetic per path, with the detectors' shipped cutoffs + (AnomalyThresholds): robust modified-z 3.5 → the escape opens at 10.5σ; heavy-tail modified-z 5.0 + (waits, query duration) → 15σ; classical z 2.0 (rollup-bound metrics, pre-#1743 facts) → 6σ — the + #1743 fleet measurement read a busy tenant's REAL 2-3x evening surge at 1.4-2.0 classical sigmas, + so 6 classical sigmas against a stddev that history has already inflated is a genuinely rare + reading, not a busy evening. All three sit under the 25σ display cap (SigmaDisplayCap), so a fact + can actually carry them. The never-blind fallback path (baseline_low_quality) has no meaningful + sigma, so it escapes only at 3x its ABSOLUTE bar (fallback_exceedance >= 3): I/O latency 150 ms, + batch requests 15,000/s, sessions 1,500, query duration 15 s total elapsed — and CPU (bar 90%) and + memory total/target (bar 101%) can never reach 3x their bars, so a young store's CPU or memory + anomaly cannot escape on an untrustworthy baseline at all, which is correct: "we do not know your + normal yet" is not evidence of an outage. An operator who scales a metric's deviation threshold + scales its fire_threshold with it (ModifiedZThresholdFor), so the escape bar tracks the knob. */ + private const double ExtremeAnomalyMultiple = 3.0; + /// /// Tuning-class keys whose FINAL severity is capped at the WARNING ceiling (Layer 3) unless an /// impact peer co-fired: parallelism (CXPACKET/CXCONSUMER), excessive-DOP queries, and every - /// anomaly fact. Today only CXPACKET can exceed the ceiling on amplifiers (ANOMALY_* and - /// QUERY_HIGH_DOP already max at 1.0) — the rest is forward-safety as those ramps evolve. + /// anomaly fact. CXPACKET and (#3526) the corroborated ANOMALY_* families can exceed the ceiling + /// on amplifiers; an anomaly is released from the cap only when holds. + /// QUERY_HIGH_DOP still maxes at 1.0 — its membership is forward-safety as that ramp evolves. /// private static bool IsTuningClassKey(string key) => key is "CXPACKET" or "CXCONSUMER" or "QUERY_HIGH_DOP" || key.StartsWith("ANOMALY_", StringComparison.OrdinalIgnoreCase); + /// + /// The deviation-scored anomaly families (the z-score / modified-z detectors: peak vs a per-server + /// hour-of-week baseline, graded off deviation_sigma against fire_threshold, or off + /// fallback_exceedance on the low-quality path). Shared by and + /// so the two cannot route a key differently. + /// + private static bool IsDeviationScoredAnomalyKey(string key) => + key.StartsWith("ANOMALY_CPU_SPIKE", StringComparison.OrdinalIgnoreCase) + || key.StartsWith("ANOMALY_READ_LATENCY", StringComparison.OrdinalIgnoreCase) + || key.StartsWith("ANOMALY_WRITE_LATENCY", StringComparison.OrdinalIgnoreCase) + || key.StartsWith("ANOMALY_BATCH_REQUESTS", StringComparison.OrdinalIgnoreCase) + || key.StartsWith("ANOMALY_SESSION_SPIKE", StringComparison.OrdinalIgnoreCase) + || key.StartsWith("ANOMALY_QUERY_DURATION", StringComparison.OrdinalIgnoreCase) + || key.StartsWith("ANOMALY_MEMORY_PRESSURE", StringComparison.OrdinalIgnoreCase); + + /// + /// #1743: the cutoff a deviation-scored anomaly actually FIRED at (carried by the detector as + /// fire_threshold): the knob-scaled classical threshold on the classical path, the knob-scaled + /// modified-z cutoff on the robust path, and the pre-#1743 default of 2.0 when the fact carries none. + /// The severity ramp anchors here and the extremity escape is a multiple of it. + /// + private static double FireAnchor(Fact fact) + { + var anchor = fact.Metadata.GetValueOrDefault("fire_threshold", 2.0); + return anchor <= 0 ? 2.0 : anchor; + } + + /// + /// #3526: whether an ANOMALY_* fact's deviation is extreme enough to leave the Layer-3 tuning-class + /// cap on its own evidence (see for the arithmetic). Routes + /// each family off the SAME metadata its severity ramp grades from, so a fact can never be extreme + /// on one statistic while scored on another: + /// + /// Deviation-scored families, trustworthy baseline: deviation_sigma >= 3 x fire_threshold. + /// Deviation-scored families, low-quality baseline: fallback_exceedance >= 3 — the + /// stored sigma is the real (small, meaningless) z the detector refused to trust, so the escape + /// must not read it either. + /// ANOMALY_WAIT_PROFILE on the robust trigger: modified_z >= 3 x 5.0; on the + /// pre-#1743 / robust-less ratio trigger: ratio >= 3 x 4.0. Never for is_new — a + /// first-occurrence profile's sentinel ratio (NoBaselineRatio, 100) is a scoring device, not a + /// measurement, and "no baseline" cannot be "extreme against baseline". + /// + /// The ratio/count/delta families — blocking and deadlock spikes, day-over-day object growth and + /// contention, the legacy per-type ANOMALY_WAIT_ facts — stay capped: none is graded in sigmas, so + /// "3x the fire threshold" has no calibrated meaning for them, and the blocking/deadlock classes + /// already reach CRITICAL through their never-capped impact keys (BLOCKING_EVENTS, BLOCKING_CHAIN, + /// DEADLOCKS) when the events are real. + /// + private static bool IsExtremeAnomaly(Fact fact) + { + if (!fact.Key.StartsWith("ANOMALY_", StringComparison.OrdinalIgnoreCase)) return false; + + if (IsDeviationScoredAnomalyKey(fact.Key)) + { + if (fact.Metadata.GetValueOrDefault("baseline_low_quality") >= 1.0) + return fact.Metadata.GetValueOrDefault("fallback_exceedance") >= ExtremeAnomalyMultiple; + + return fact.Metadata.GetValueOrDefault("deviation_sigma") >= ExtremeAnomalyMultiple * FireAnchor(fact); + } + + if (fact.Key.StartsWith("ANOMALY_WAIT_PROFILE", StringComparison.OrdinalIgnoreCase)) + { + if (fact.Metadata.GetValueOrDefault("is_new") > 0) return false; + var modifiedZ = fact.Metadata.GetValueOrDefault("modified_z"); + if (modifiedZ > 0) + return modifiedZ >= ExtremeAnomalyMultiple * Baselines.AnomalyThresholds.HeavyTailModifiedZThreshold; + return fact.Metadata.GetValueOrDefault("ratio") >= ExtremeAnomalyMultiple * WaitProfileRatioFloor; + } + + return false; + } + /// /// Scores anomaly facts based on deviation from baseline. /// At 2σ → 0.5, at 4σ → 1.0. Higher deviations are more severe. @@ -616,13 +726,7 @@ key is "CXPACKET" or "CXCONSUMER" or "QUERY_HIGH_DOP" /// private static double ScoreAnomalyFact(Fact fact) { - if (fact.Key.StartsWith("ANOMALY_CPU_SPIKE", StringComparison.OrdinalIgnoreCase) - || fact.Key.StartsWith("ANOMALY_READ_LATENCY", StringComparison.OrdinalIgnoreCase) - || fact.Key.StartsWith("ANOMALY_WRITE_LATENCY", StringComparison.OrdinalIgnoreCase) - || fact.Key.StartsWith("ANOMALY_BATCH_REQUESTS", StringComparison.OrdinalIgnoreCase) - || fact.Key.StartsWith("ANOMALY_SESSION_SPIKE", StringComparison.OrdinalIgnoreCase) - || fact.Key.StartsWith("ANOMALY_QUERY_DURATION", StringComparison.OrdinalIgnoreCase) - || fact.Key.StartsWith("ANOMALY_MEMORY_PRESSURE", StringComparison.OrdinalIgnoreCase)) + if (IsDeviationScoredAnomalyKey(fact.Key)) { // Deviation-based scoring: 2σ = 0.5, 4σ = 1.0 var deviation = fact.Metadata.GetValueOrDefault("deviation_sigma"); @@ -651,8 +755,7 @@ private static double ScoreAnomalyFact(Fact fact) shape for classical fires and for pre-#1743 facts (default 2.0), and the same proportional shape for robust fires at 3.5 or 5.0. Without the anchor, a family firing at 5σ scores saturated-flat 1.0 forever against a ramp built for 2σ fires. */ - var anchor = fact.Metadata.GetValueOrDefault("fire_threshold", 2.0); - if (anchor <= 0) anchor = 2.0; + var anchor = FireAnchor(fact); if (deviation < anchor) return 0.0; var base_score = 0.5 + 0.5 * Math.Min((deviation - anchor) / anchor, 1.0); return base_score * confidence; @@ -771,10 +874,252 @@ private static List GetAmplifiers(Fact fact) "PLAN_REGRESSION" => PlanRegressionAmplifiers(), "DB_CONFIG" => DbConfigAmplifiers(), "DISK_SPACE" => DiskSpaceAmplifiers(), + _ when fact.Key.StartsWith("ANOMALY_", StringComparison.OrdinalIgnoreCase) => AnomalyAmplifiers(fact.Key), _ => [] }; } + /// + /// #3526: the anomaly co-fire arm — corroboration for the baseline engine's findings, which had no + /// amplifiers at all and so could never leave their 1.0 base. Modelled on the impact-peer style of + /// the Layer-3 escape and the PAGEIOLATCH / IO-latency arms: a sibling anomaly family firing in the + /// same window (BaseSeverity > 0 — the scorer zeroes anything under its cutoff, so > 0 means + /// "fired against its own baseline") is a co-fire, and a MEASURED absolute fact confirming the same + /// pressure (SQL CPU >= 80%, the I/O-latency fact at its concerning bar, grant waiters, the + /// buffer-pool / log waits at the bars their own arms use) is a co-fire. + /// + /// Worked numbers — the arm's magnitudes are chosen so corroboration can carry an EXTREME anomaly + /// past the 1.5 notify floor and nothing can carry a routine one there: + /// + /// Base at the fire threshold (0.5) with two co-fires: 0.5 x (1 + 0.3 + 0.3) = 0.8. With every + /// arm lit (the load family's maximum is +1.2): 1.1. Never reaches 1.5, and the Layer-3 cap holds + /// regardless because the anomaly is not extreme. + /// Base saturated but ROUTINE (2x the anchor — 4σ classical, 7σ robust; 1.0) with three co-fires: + /// 1.0 x 1.9 = 1.9 → capped to 1.49. Still WARNING: saturation is not extremity, and the cap is + /// exactly what keeps a busy evening from paging. + /// Base EXTREME (>= 3x the anchor; 1.0, the cap released) alone: 1.0. Below the cap it escaped + /// — a lone 20σ reading with nothing else moving does not page, by design: the CRITICAL band is + /// earned only with corroboration (the same rule every other base fact follows), and a solitary + /// extreme reading is exactly the shape a collector hiccup or a variance-collapsed baseline pinned + /// at the 25σ display cap produces. + /// Base EXTREME with one +0.3 co-fire: 1.3, WARNING. With two: 1.6 → pages. The issue's own + /// 3am shape — a 20σ session spike (root, extreme) beside a 15σ batch-request anomaly (+0.3) and + /// SQL CPU at 85% (+0.3) — scores 1.6 and reaches the operator for the first time at shipped + /// settings. A +0.3 and a +0.2 land on 1.5 exactly: two independent corroborators is the bar. + /// + /// + private static List AnomalyAmplifiers(string key) + { + if (key.StartsWith("ANOMALY_SESSION_SPIKE", StringComparison.OrdinalIgnoreCase) + || key.StartsWith("ANOMALY_BATCH_REQUESTS", StringComparison.OrdinalIgnoreCase) + || key.StartsWith("ANOMALY_CPU_SPIKE", StringComparison.OrdinalIgnoreCase) + || key.StartsWith("ANOMALY_QUERY_DURATION", StringComparison.OrdinalIgnoreCase)) + return LoadAnomalyAmplifiers(key); + + if (key.StartsWith("ANOMALY_READ_LATENCY", StringComparison.OrdinalIgnoreCase)) + return ReadLatencyAnomalyAmplifiers(); + + if (key.StartsWith("ANOMALY_WRITE_LATENCY", StringComparison.OrdinalIgnoreCase)) + return WriteLatencyAnomalyAmplifiers(); + + if (key.StartsWith("ANOMALY_WAIT_PROFILE", StringComparison.OrdinalIgnoreCase)) + return WaitProfileAnomalyAmplifiers(); + + if (key.StartsWith("ANOMALY_MEMORY_PRESSURE", StringComparison.OrdinalIgnoreCase)) + return MemoryPressureAnomalyAmplifiers(); + + // Blocking/deadlock spikes and the object-stats anomalies have no arm: they are not released from + // the cap (IsExtremeAnomaly) and their impact lives in the never-capped BLOCKING_* / DEADLOCKS keys. + return []; + } + + /// + /// A sibling anomaly family fired in the same window against its own baseline. The self-key is + /// skipped by the callers, so a family never corroborates itself. + /// + private static bool AnomalyCoFired(Dictionary facts, string siblingKey) => + facts.TryGetValue(siblingKey, out var sibling) && sibling.BaseSeverity > 0; + + /// + /// The LOAD family — sessions, batch requests, CPU, query duration — corroborate one another (a real + /// surge moves more than one of them) and are confirmed by measured SQL CPU at the 80% bar the SOS + /// and compile-gateway arms already use. Each sibling is +0.3; the root's own key is omitted. + /// + private static List LoadAnomalyAmplifiers(string selfKey) + { + var amplifiers = new List(); + void Sibling(string siblingKey, string description) + { + if (selfKey.StartsWith(siblingKey, StringComparison.OrdinalIgnoreCase)) return; + amplifiers.Add(new() + { + Description = description, + Boost = 0.3, + Predicate = facts => AnomalyCoFired(facts, siblingKey) + }); + } + + Sibling("ANOMALY_SESSION_SPIKE", "Session-count anomaly co-fired — the surge is visible in connections too"); + Sibling("ANOMALY_BATCH_REQUESTS", "Batch-request anomaly co-fired — the surge is visible in throughput too"); + Sibling("ANOMALY_CPU_SPIKE", "CPU anomaly co-fired — the surge is consuming CPU far above this server's norm"); + Sibling("ANOMALY_QUERY_DURATION", "Query-duration anomaly co-fired — the surge is slowing queries"); + amplifiers.Add(new() + { + Description = "SQL Server CPU >= 80% — the surge is consuming real CPU, not just moving a counter", + Boost = 0.3, + Predicate = facts => facts.TryGetValue("CPU_SQL_PERCENT", out var cpu) && cpu.Value >= 80 + }); + return amplifiers; + } + + /// + /// ANOMALY_READ_LATENCY: a per-server read-latency deviation confirmed by the absolute read-latency fact + /// at its concerning bar (20 ms — "bad in absolute terms, not only for you"), by the wait profile + /// shifting (queries are actually waiting on it), by write latency deviating alongside (a storage-side + /// event, not one hot file), and by PAGEIOLATCH at the IO_READ_LATENCY_MS arm's own 10% bar. + /// + private static List ReadLatencyAnomalyAmplifiers() => + [ + new() + { + Description = "Read latency at the absolute concerning bar — slow for any server, not only against this baseline", + Boost = 0.3, + Predicate = facts => facts.TryGetValue("IO_READ_LATENCY_MS", out var io) && io.BaseSeverity >= 0.5 + }, + new() + { + Description = "Wait-profile anomaly co-fired — queries are waiting on the slow reads", + Boost = 0.3, + Predicate = facts => AnomalyCoFired(facts, "ANOMALY_WAIT_PROFILE") + }, + new() + { + Description = "Write-latency anomaly co-fired — the storage path is slow in both directions", + Boost = 0.3, + Predicate = facts => AnomalyCoFired(facts, "ANOMALY_WRITE_LATENCY") + }, + new() + { + Description = "PAGEIOLATCH waits elevated — buffer pool misses confirm the read pressure", + Boost = 0.2, + Predicate = facts => HasSignificantWait(facts, "PAGEIOLATCH_SH", 0.10) + || HasSignificantWait(facts, "PAGEIOLATCH_EX", 0.10) + } + ]; + + /// + /// ANOMALY_WRITE_LATENCY: the write-side twin — the absolute write-latency fact at its concerning bar + /// (10 ms), the wait profile shifting, read latency deviating alongside, and WRITELOG at the + /// IO_WRITE_LATENCY_MS arm's own 5% bar. + /// + private static List WriteLatencyAnomalyAmplifiers() => + [ + new() + { + Description = "Write latency at the absolute concerning bar — slow for any server, not only against this baseline", + Boost = 0.3, + Predicate = facts => facts.TryGetValue("IO_WRITE_LATENCY_MS", out var io) && io.BaseSeverity >= 0.5 + }, + new() + { + Description = "Wait-profile anomaly co-fired — queries are waiting on the slow writes", + Boost = 0.3, + Predicate = facts => AnomalyCoFired(facts, "ANOMALY_WAIT_PROFILE") + }, + new() + { + Description = "Read-latency anomaly co-fired — the storage path is slow in both directions", + Boost = 0.3, + Predicate = facts => AnomalyCoFired(facts, "ANOMALY_READ_LATENCY") + }, + new() + { + Description = "WRITELOG waits elevated — transaction log I/O confirms the write pressure", + Boost = 0.2, + Predicate = facts => HasSignificantWait(facts, "WRITELOG", 0.05) + } + ]; + + /// + /// ANOMALY_WAIT_PROFILE: the all-types wait rate shifting against its baseline, corroborated by WHAT the + /// waiting is costing — I/O latency deviating (+0.3 each side), query duration deviating (+0.3: the + /// waits are landing on user queries), and the load family moving (+0.2 each: a surge is driving it). + /// + private static List WaitProfileAnomalyAmplifiers() => + [ + new() + { + Description = "Read-latency anomaly co-fired — the wait shift is storage-bound", + Boost = 0.3, + Predicate = facts => AnomalyCoFired(facts, "ANOMALY_READ_LATENCY") + }, + new() + { + Description = "Write-latency anomaly co-fired — the wait shift is log/storage-bound", + Boost = 0.3, + Predicate = facts => AnomalyCoFired(facts, "ANOMALY_WRITE_LATENCY") + }, + new() + { + Description = "Query-duration anomaly co-fired — the waits are landing on user queries", + Boost = 0.3, + Predicate = facts => AnomalyCoFired(facts, "ANOMALY_QUERY_DURATION") + }, + new() + { + Description = "CPU anomaly co-fired — a load surge is driving the wait shift", + Boost = 0.2, + Predicate = facts => AnomalyCoFired(facts, "ANOMALY_CPU_SPIKE") + }, + new() + { + Description = "Session-count anomaly co-fired — a connection surge is driving the wait shift", + Boost = 0.2, + Predicate = facts => AnomalyCoFired(facts, "ANOMALY_SESSION_SPIKE") + } + ]; + + /// + /// ANOMALY_MEMORY_PRESSURE: total-over-target deviating against baseline, corroborated by the symptoms + /// real memory pressure produces — ring-buffer pressure notifications (the engine saying so itself), + /// grant waiters at the PAGEIOLATCH arm's bar, RESOURCE_SEMAPHORE in the wait stats, PAGEIOLATCH at + /// the 10% bar (buffer pool churn), and read latency deviating (the churn reaching storage). + /// + private static List MemoryPressureAnomalyAmplifiers() => + [ + new() + { + Description = "Memory-pressure notifications present — the engine itself is reporting pressure", + Boost = 0.3, + Predicate = facts => facts.TryGetValue("MEMORY_PRESSURE_EVENTS", out var mp) && mp.BaseSeverity > 0 + }, + new() + { + Description = "Memory grant waiters present — grants competing for the same memory", + Boost = 0.3, + Predicate = facts => facts.TryGetValue("MEMORY_GRANT_PENDING", out var mg) && mg.Value >= 1 + }, + new() + { + Description = "RESOURCE_SEMAPHORE waits present — grant pressure visible in wait stats", + Boost = 0.2, + Predicate = facts => facts.TryGetValue("RESOURCE_SEMAPHORE", out var rs) && rs.BaseSeverity > 0 + }, + new() + { + Description = "PAGEIOLATCH waits elevated — buffer pool churning under the pressure", + Boost = 0.2, + Predicate = facts => HasSignificantWait(facts, "PAGEIOLATCH_SH", 0.10) + || HasSignificantWait(facts, "PAGEIOLATCH_EX", 0.10) + }, + new() + { + Description = "Read-latency anomaly co-fired — the churn is reaching storage", + Boost = 0.2, + Predicate = facts => AnomalyCoFired(facts, "ANOMALY_READ_LATENCY") + } + ]; + /// /// PARAMETER_SENSITIVITY: a single plan with wildly varying per-execution cost. /// Corroborated by grant/spill divergence and memory-grant pressure. From 20d0076a63969829b073c16d3f0fc8d491b2e3aa Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 18 Sep 2026 11:04:08 -0400 Subject: [PATCH 2/2] Bound the extremity escape bar at SigmaDisplayCap (review on #3584) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AnomalyGate clamps the stored deviation_sigma at 25σ before the scorer sees it, so an operator-scaled anchor above ~8.3 made 3x unreachable and the escape silently dead for that metric — the defect this PR fixes, for a tuned store. IsExtremeAnomaly now takes min(3x anchor, SigmaDisplayCap); two Theory rows pin anchor 10.0 at 25.0 (released) and 24.99 (capped). --- Lite.Tests/FactScorerTests.cs | 5 +++++ PerformanceMonitor.Analysis/FactScorer.cs | 14 ++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/Lite.Tests/FactScorerTests.cs b/Lite.Tests/FactScorerTests.cs index 93f2dd9d10..8ec9d476d1 100644 --- a/Lite.Tests/FactScorerTests.cs +++ b/Lite.Tests/FactScorerTests.cs @@ -556,6 +556,9 @@ public void Score_SaturatedButRoutineAnomaly_WithThreeCoFires_CappedAtWarningCei // (a pre-#1743 fact with no fire_threshold) → 6σ; robust 3.5 → 10.5σ; heavy-tail 5.0 → 15σ. Each // row plants the root beside the same two co-fires (+0.3 +0.3 → 1.6 when released, 1.49 when // capped) and probes one side of the bar. The root's base is 1.0 in every row (all are >= 2x anchor). + // The last two rows pin the display-cap bound (review-caught on this PR): AnomalyGate clamps the + // stored deviation_sigma at SigmaDisplayCap (25σ), so an operator-scaled anchor of 10.0 (3x = 30σ, + // unreachable) must open at 25σ instead of going silently dead — and a hair under 25 stays capped. [Theory] [InlineData(6.0, 0.0, 1.6)] // classical: 3 x 2.0 → released [InlineData(5.99, 0.0, 1.49)] // classical: a hair under → capped @@ -563,6 +566,8 @@ public void Score_SaturatedButRoutineAnomaly_WithThreeCoFires_CappedAtWarningCei [InlineData(10.49, 3.5, 1.49)] // robust: a hair under → capped [InlineData(15.0, 5.0, 1.6)] // heavy-tail: 3 x 5.0 → released [InlineData(14.99, 5.0, 1.49)] // heavy-tail: a hair under → capped + [InlineData(25.0, 10.0, 1.6)] // hard-tuned anchor: min(30, 25) = the display cap → released + [InlineData(24.99, 10.0, 1.49)] // hard-tuned anchor: a hair under the cap → capped public void Score_AnomalyExtremityEscape_OpensAtThreeTimesTheFireThreshold( double deviationSigma, double fireThreshold, double expected) { diff --git a/PerformanceMonitor.Analysis/FactScorer.cs b/PerformanceMonitor.Analysis/FactScorer.cs index aa4f6f12e9..9011a5cf7d 100644 --- a/PerformanceMonitor.Analysis/FactScorer.cs +++ b/PerformanceMonitor.Analysis/FactScorer.cs @@ -633,7 +633,15 @@ so 6 classical sigmas against a stddev that history has already inflated is a ge memory total/target (bar 101%) can never reach 3x their bars, so a young store's CPU or memory anomaly cannot escape on an untrustworthy baseline at all, which is correct: "we do not know your normal yet" is not evidence of an outage. An operator who scales a metric's deviation threshold - scales its fire_threshold with it (ModifiedZThresholdFor), so the escape bar tracks the knob. */ + scales its fire_threshold with it (ModifiedZThresholdFor), so the escape bar tracks the knob — up + to the display cap: AnomalyGate clamps the stored deviation_sigma at SigmaDisplayCap (25σ) BEFORE + the scorer ever sees it, so a bar above 25σ would be unreachable and the escape would go silently + dead for exactly the deployments that tuned a metric hard (a knob past ~8.3x the shipped anchor + puts 3x over 25) — the same "structurally quietest" defect this constant exists to fix, just for a + differently-tuned store. IsExtremeAnomaly therefore takes min(3x anchor, SigmaDisplayCap): a sigma + pinned at the cap means "at least 25σ", which is extreme under any anchor an operator can set. The + wait profile's modified_z is not display-capped (BaselineMath.ModifiedZScore) and needs no such + bound. */ private const double ExtremeAnomalyMultiple = 3.0; /// @@ -704,7 +712,9 @@ private static bool IsExtremeAnomaly(Fact fact) if (fact.Metadata.GetValueOrDefault("baseline_low_quality") >= 1.0) return fact.Metadata.GetValueOrDefault("fallback_exceedance") >= ExtremeAnomalyMultiple; - return fact.Metadata.GetValueOrDefault("deviation_sigma") >= ExtremeAnomalyMultiple * FireAnchor(fact); + // Bounded at the display cap — see ExtremeAnomalyMultiple: the stored sigma can never exceed it. + var escapeBar = Math.Min(ExtremeAnomalyMultiple * FireAnchor(fact), Baselines.AnomalyThresholds.SigmaDisplayCap); + return fact.Metadata.GetValueOrDefault("deviation_sigma") >= escapeBar; } if (fact.Key.StartsWith("ANOMALY_WAIT_PROFILE", StringComparison.OrdinalIgnoreCase))