Network-dependent tests skip loudly on feed outage; watchdog measures monotonic time (#679)

src/MeshWeaver.Hosting.Monolith.TestBase/MonolithMeshTestBase.cs

@@ -653,6 +653,16 @@ private static void TestMemTrace(string testClass, string phase, bool forceGc)
     // past the deadline and xUnit eventually reports Passed with the actual
     // (multi-minute) duration. The watchdog below catches that uniformly.
     private DateTimeOffset _testMethodStartedAt;
+    // Monotonic twin of the wall-clock stamp above (#679): wall-clock keeps counting
+    // through host suspension (laptop sleep, frozen runner) — a window in which the
+    // test could not have made progress — and the sweep of 2026-07 recorded "hangs" of
+    // 5523 s and 7288 s against a 720 s cap that all passed under caffeinate. Stopwatch
+    // does not advance across suspension on Linux (CLOCK_MONOTONIC) and Intel macOS
+    // (mach_absolute_time), so deadlines are enforced on it; on Apple-Silicon macOS the
+    // OS monotonic clock may tick through sleep, making the divergence report below
+    // best-effort there — but enforcement can never be WORSE than the wall clock it
+    // replaces.
+    private Stopwatch? _testMethodStopwatch;
     /// <summary>Soft cap — anything above this gets a warning in the test log.</summary>
     protected virtual TimeSpan TestSoftDeadline => TimeSpan.FromSeconds(30);
     /// <summary>
@@ -793,6 +803,7 @@ public override async ValueTask InitializeAsync()
             // compute actual test-method duration and apply the soft/hard
             // deadlines (see TestSoftDeadline / TestHardDeadline).
             _testMethodStartedAt = DateTimeOffset.UtcNow;
+            _testMethodStopwatch = Stopwatch.StartNew();
         }
         catch (Exception ex)
         {
@@ -1186,9 +1197,24 @@ public override async ValueTask DisposeAsync()
         // [Fact(Timeout=N)] is cooperative — a test that ignores the ct
         // happily blocks past its declared timeout and gets reported as
         // Passed. We catch every such silent deadlock here, uniformly.
-        TimeSpan? testMethodElapsed = _testMethodStartedAt == default
+        TimeSpan? wallElapsed = _testMethodStartedAt == default
             ? null
             : DateTimeOffset.UtcNow - _testMethodStartedAt;
+        var monotonicElapsed = _testMethodStopwatch?.Elapsed;
+        // Enforce on the monotonic clock (#679): the wall clock includes host-suspension
+        // windows in which the test could not have made progress — failing it for that
+        // time misattributes a sleep/stall to the test's CancellationToken handling.
+        var testMethodElapsed = monotonicElapsed ?? wallElapsed;
+        if (wallElapsed is { } wall && monotonicElapsed is { } mono &&
+            wall - mono > TimeSpan.FromSeconds(10))
+        {
+            FileOutput.WriteLine(
+                $"[WATCHDOG-SUSPEND] {testName}: wall-clock elapsed {wall.TotalSeconds:F1}s vs " +
+                $"monotonic {mono.TotalSeconds:F1}s — the host was suspended ~" +
+                $"{(wall - mono).TotalSeconds:F0}s during this test (laptop sleep / runner stall). " +
+                "Deadlines are enforced on the monotonic clock, so the suspension window alone " +
+                "cannot fail the test.");
+        }
         if (testMethodElapsed is { } elapsed)
         {
             var hardDeadline = EffectiveHardDeadline;

test/MeshWeaver.Graph.Test/NuGetAssemblyResolverTest.cs

@@ -1,5 +1,7 @@
 using System;
 using System.IO;
+using System.Net.Http;
+using System.Threading;
 using System.Threading.Tasks;
 using MeshWeaver.NuGet;
 using Microsoft.Extensions.Logging.Abstractions;
@@ -8,19 +10,46 @@
 namespace MeshWeaver.Graph.Test;
 
 /// <summary>
-/// End-to-end test of NuGet restore against api.nuget.org. Requires network access.
-/// Disable with environment variable MESHWEAVER_SKIP_NUGET=1.
+/// End-to-end test of NuGet restore against api.nuget.org. Requires network access:
+/// each test probes the live feed first and SKIPS with an explicit reason when it is
+/// unreachable (#679) — a feed outage must present as a skip, never as a red that
+/// points at NuGet internals. Disable outright with MESHWEAVER_SKIP_NUGET=1.
 /// </summary>
 [Collection("NuGetNetwork")]
 public class NuGetAssemblyResolverTest
 {
-    private static bool ShouldSkip =>
-        Environment.GetEnvironmentVariable("MESHWEAVER_SKIP_NUGET") == "1";
+    /// <summary>
+    /// Loud gate for the live-feed dependency (#679): when nuget.org has a bad minute
+    /// these tests used to fail with a FatalProtocolException naming NuGet internals —
+    /// a phantom flake someone then triages. Skipping with the cause on the result is
+    /// honest; a silent early-return would be a vacuous pass. Probed per test on
+    /// purpose: no cached static state, and the serialized collection makes the cost
+    /// one round-trip per test against 180 s budgets.
+    /// </summary>
+    private static async Task SkipUnlessNuGetOrgReachable(CancellationToken ct)
+    {
+        Assert.SkipWhen(Environment.GetEnvironmentVariable("MESHWEAVER_SKIP_NUGET") == "1",
+            "NuGet end-to-end tests disabled via MESHWEAVER_SKIP_NUGET=1");
+        using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(10) };
+        try
+        {
+            using var response = await http.GetAsync("https://api.nuget.org/v3/index.json", ct);
+            Assert.SkipUnless(response.IsSuccessStatusCode,
+                $"api.nuget.org service index answered {(int)response.StatusCode} — feed unhealthy; " +
+                "this end-to-end test needs the live feed (#679)");
+        }
+        catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
+        {
+            Assert.Skip($"api.nuget.org unreachable ({ex.GetType().Name}: {ex.Message}) — this " +
+                "end-to-end test needs the live feed; a network outage presents as a SKIP, not a " +
+                "phantom test failure (#679)");
+        }
+    }
 
     [Fact(Timeout = 180_000)]
     public async Task Resolve_Humanizer_ReturnsExistingDllPaths()
     {
-        if (ShouldSkip) return;
+        await SkipUnlessNuGetOrgReachable(TestContext.Current.CancellationToken);
         var resolver = new NuGetAssemblyResolver(NullLogger<NuGetAssemblyResolver>.Instance);
 
         var result = await resolver.ResolveAsync(
@@ -37,7 +66,7 @@ [new NuGetPackageReference("Humanizer", "2.14.1")],
     [Fact(Timeout = 180_000)]
     public async Task Resolve_MathNetNumerics_LoadsTransitiveDeps()
     {
-        if (ShouldSkip) return;
+        await SkipUnlessNuGetOrgReachable(TestContext.Current.CancellationToken);
         var resolver = new NuGetAssemblyResolver(NullLogger<NuGetAssemblyResolver>.Instance);
 
         var result = await resolver.ResolveAsync(
@@ -53,7 +82,7 @@ [new NuGetPackageReference("MathNet.Numerics", "5.0.0")],
     [Fact(Timeout = 30_000)]
     public async Task Resolve_UnknownPackage_Throws()
     {
-        if (ShouldSkip) return;
+        await SkipUnlessNuGetOrgReachable(TestContext.Current.CancellationToken);
         var resolver = new NuGetAssemblyResolver(NullLogger<NuGetAssemblyResolver>.Instance);
 
         var act = () => resolver.ResolveAsync(
@@ -66,7 +95,7 @@ [new NuGetPackageReference("This.Package.Does.Not.Exist.Really", "1.0.0")],
     [Fact(Timeout = 180_000)]
     public async Task Resolve_TwiceWithSameInputs_HitsCache()
     {
-        if (ShouldSkip) return;
+        await SkipUnlessNuGetOrgReachable(TestContext.Current.CancellationToken);
         var resolver = new NuGetAssemblyResolver(NullLogger<NuGetAssemblyResolver>.Instance);
         var refs = new[] { new NuGetPackageReference("Humanizer", "2.14.1") };
 

test/MeshWeaver.MemexTemplate.Test/MemexTemplateGeneratorTest.cs

@@ -3,6 +3,7 @@
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
+using System.Net.Http;
 using System.Threading.Tasks;
 using System.Xml.Linq;
 using Xunit;
@@ -30,8 +31,37 @@ public void Dispose()
     private string GeneratorScript =>
         Path.Combine(RepoRoot, "tools", "generate-memex-template.cs");
 
+    /// <summary>
+    /// Loud gate for the hidden network dependency (#679): <c>dotnet run script.cs</c>
+    /// implicitly RESTORES the file-based app, so an unreachable nuget.org fails every
+    /// test here with an NU1900 that points at NuGet internals — a phantom flake someone
+    /// then triages. Skipping with the cause on the result is honest. (Deliberately
+    /// duplicated from NuGetAssemblyResolverTest — this project is standalone, with no
+    /// shared test-utility reference to host a common helper.)
+    /// </summary>
+    private static async Task SkipUnlessNuGetOrgReachable()
+    {
+        using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(10) };
+        try
+        {
+            using var response = await http.GetAsync("https://api.nuget.org/v3/index.json",
+                TestContext.Current.CancellationToken);
+            Assert.SkipUnless(response.IsSuccessStatusCode,
+                $"api.nuget.org service index answered {(int)response.StatusCode} — feed unhealthy; " +
+                "the generator's implicit restore needs the live feed (#679)");
+        }
+        catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
+        {
+            Assert.Skip($"api.nuget.org unreachable ({ex.GetType().Name}: {ex.Message}) — the " +
+                "generator's implicit `dotnet run` restore needs the live feed; a network outage " +
+                "presents as a SKIP, not a phantom test failure (#679)");
+        }
+    }
+
     private async Task RunGenerator(string version = "0.0.0-test")
     {
+        await SkipUnlessNuGetOrgReachable();
+
         var psi = new ProcessStartInfo("dotnet", $"run \"{GeneratorScript}\" -- {version} \"{RepoRoot}\" \"{_outputPath}\"")
         {
             WorkingDirectory = RepoRoot,
@@ -44,6 +74,19 @@ private async Task RunGenerator(string version = "0.0.0-test")
         var stderr = await process.StandardError.ReadToEndAsync();
         await process.WaitForExitAsync();
 
+        // The feed can die BETWEEN the probe above and the restore. NU1900 / "unable to
+        // load the service index" name exactly that condition and nothing the generator
+        // itself can cause — so it is the outage skip, not a masked generator defect.
+        if (process.ExitCode != 0)
+        {
+            var combined = stdout + stderr;
+            Assert.SkipWhen(
+                combined.Contains("NU1900") ||
+                combined.Contains("Unable to load the service index", StringComparison.OrdinalIgnoreCase),
+                "nuget.org became unreachable during the generator's implicit restore — network " +
+                $"outage, not a generator defect (#679).\nstdout: {stdout}\nstderr: {stderr}");
+        }
+
         process.ExitCode.Should().Be(0, $"generator failed.\nstdout: {stdout}\nstderr: {stderr}");
         _lastStdout = stdout;
         _lastStderr = stderr;