diff --git a/src/MeshWeaver.Hosting.Monolith.TestBase/MonolithMeshTestBase.cs b/src/MeshWeaver.Hosting.Monolith.TestBase/MonolithMeshTestBase.cs index 5b16d54234..2548151cfe 100644 --- a/src/MeshWeaver.Hosting.Monolith.TestBase/MonolithMeshTestBase.cs +++ b/src/MeshWeaver.Hosting.Monolith.TestBase/MonolithMeshTestBase.cs @@ -653,6 +653,16 @@ private static void TestMemTrace(string testClass, string phase, bool forceGc) // past the deadline and xUnit eventually reports Passed with the actual // (multi-minute) duration. The watchdog below catches that uniformly. private DateTimeOffset _testMethodStartedAt; + // Monotonic twin of the wall-clock stamp above (#679): wall-clock keeps counting + // through host suspension (laptop sleep, frozen runner) — a window in which the + // test could not have made progress — and the sweep of 2026-07 recorded "hangs" of + // 5523 s and 7288 s against a 720 s cap that all passed under caffeinate. Stopwatch + // does not advance across suspension on Linux (CLOCK_MONOTONIC) and Intel macOS + // (mach_absolute_time), so deadlines are enforced on it; on Apple-Silicon macOS the + // OS monotonic clock may tick through sleep, making the divergence report below + // best-effort there — but enforcement can never be WORSE than the wall clock it + // replaces. + private Stopwatch? _testMethodStopwatch; /// Soft cap — anything above this gets a warning in the test log. protected virtual TimeSpan TestSoftDeadline => TimeSpan.FromSeconds(30); /// @@ -793,6 +803,7 @@ public override async ValueTask InitializeAsync() // compute actual test-method duration and apply the soft/hard // deadlines (see TestSoftDeadline / TestHardDeadline). _testMethodStartedAt = DateTimeOffset.UtcNow; + _testMethodStopwatch = Stopwatch.StartNew(); } catch (Exception ex) { @@ -1186,9 +1197,24 @@ public override async ValueTask DisposeAsync() // [Fact(Timeout=N)] is cooperative — a test that ignores the ct // happily blocks past its declared timeout and gets reported as // Passed. We catch every such silent deadlock here, uniformly. - TimeSpan? testMethodElapsed = _testMethodStartedAt == default + TimeSpan? wallElapsed = _testMethodStartedAt == default ? null : DateTimeOffset.UtcNow - _testMethodStartedAt; + var monotonicElapsed = _testMethodStopwatch?.Elapsed; + // Enforce on the monotonic clock (#679): the wall clock includes host-suspension + // windows in which the test could not have made progress — failing it for that + // time misattributes a sleep/stall to the test's CancellationToken handling. + var testMethodElapsed = monotonicElapsed ?? wallElapsed; + if (wallElapsed is { } wall && monotonicElapsed is { } mono && + wall - mono > TimeSpan.FromSeconds(10)) + { + FileOutput.WriteLine( + $"[WATCHDOG-SUSPEND] {testName}: wall-clock elapsed {wall.TotalSeconds:F1}s vs " + + $"monotonic {mono.TotalSeconds:F1}s — the host was suspended ~" + + $"{(wall - mono).TotalSeconds:F0}s during this test (laptop sleep / runner stall). " + + "Deadlines are enforced on the monotonic clock, so the suspension window alone " + + "cannot fail the test."); + } if (testMethodElapsed is { } elapsed) { var hardDeadline = EffectiveHardDeadline; diff --git a/test/MeshWeaver.Graph.Test/NuGetAssemblyResolverTest.cs b/test/MeshWeaver.Graph.Test/NuGetAssemblyResolverTest.cs index 638405151b..d5a5ec32dc 100644 --- a/test/MeshWeaver.Graph.Test/NuGetAssemblyResolverTest.cs +++ b/test/MeshWeaver.Graph.Test/NuGetAssemblyResolverTest.cs @@ -1,5 +1,7 @@ using System; using System.IO; +using System.Net.Http; +using System.Threading; using System.Threading.Tasks; using MeshWeaver.NuGet; using Microsoft.Extensions.Logging.Abstractions; @@ -8,19 +10,46 @@ namespace MeshWeaver.Graph.Test; /// -/// End-to-end test of NuGet restore against api.nuget.org. Requires network access. -/// Disable with environment variable MESHWEAVER_SKIP_NUGET=1. +/// End-to-end test of NuGet restore against api.nuget.org. Requires network access: +/// each test probes the live feed first and SKIPS with an explicit reason when it is +/// unreachable (#679) — a feed outage must present as a skip, never as a red that +/// points at NuGet internals. Disable outright with MESHWEAVER_SKIP_NUGET=1. /// [Collection("NuGetNetwork")] public class NuGetAssemblyResolverTest { - private static bool ShouldSkip => - Environment.GetEnvironmentVariable("MESHWEAVER_SKIP_NUGET") == "1"; + /// + /// Loud gate for the live-feed dependency (#679): when nuget.org has a bad minute + /// these tests used to fail with a FatalProtocolException naming NuGet internals — + /// a phantom flake someone then triages. Skipping with the cause on the result is + /// honest; a silent early-return would be a vacuous pass. Probed per test on + /// purpose: no cached static state, and the serialized collection makes the cost + /// one round-trip per test against 180 s budgets. + /// + private static async Task SkipUnlessNuGetOrgReachable(CancellationToken ct) + { + Assert.SkipWhen(Environment.GetEnvironmentVariable("MESHWEAVER_SKIP_NUGET") == "1", + "NuGet end-to-end tests disabled via MESHWEAVER_SKIP_NUGET=1"); + using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(10) }; + try + { + using var response = await http.GetAsync("https://api.nuget.org/v3/index.json", ct); + Assert.SkipUnless(response.IsSuccessStatusCode, + $"api.nuget.org service index answered {(int)response.StatusCode} — feed unhealthy; " + + "this end-to-end test needs the live feed (#679)"); + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) + { + Assert.Skip($"api.nuget.org unreachable ({ex.GetType().Name}: {ex.Message}) — this " + + "end-to-end test needs the live feed; a network outage presents as a SKIP, not a " + + "phantom test failure (#679)"); + } + } [Fact(Timeout = 180_000)] public async Task Resolve_Humanizer_ReturnsExistingDllPaths() { - if (ShouldSkip) return; + await SkipUnlessNuGetOrgReachable(TestContext.Current.CancellationToken); var resolver = new NuGetAssemblyResolver(NullLogger.Instance); var result = await resolver.ResolveAsync( @@ -37,7 +66,7 @@ [new NuGetPackageReference("Humanizer", "2.14.1")], [Fact(Timeout = 180_000)] public async Task Resolve_MathNetNumerics_LoadsTransitiveDeps() { - if (ShouldSkip) return; + await SkipUnlessNuGetOrgReachable(TestContext.Current.CancellationToken); var resolver = new NuGetAssemblyResolver(NullLogger.Instance); var result = await resolver.ResolveAsync( @@ -53,7 +82,7 @@ [new NuGetPackageReference("MathNet.Numerics", "5.0.0")], [Fact(Timeout = 30_000)] public async Task Resolve_UnknownPackage_Throws() { - if (ShouldSkip) return; + await SkipUnlessNuGetOrgReachable(TestContext.Current.CancellationToken); var resolver = new NuGetAssemblyResolver(NullLogger.Instance); var act = () => resolver.ResolveAsync( @@ -66,7 +95,7 @@ [new NuGetPackageReference("This.Package.Does.Not.Exist.Really", "1.0.0")], [Fact(Timeout = 180_000)] public async Task Resolve_TwiceWithSameInputs_HitsCache() { - if (ShouldSkip) return; + await SkipUnlessNuGetOrgReachable(TestContext.Current.CancellationToken); var resolver = new NuGetAssemblyResolver(NullLogger.Instance); var refs = new[] { new NuGetPackageReference("Humanizer", "2.14.1") }; diff --git a/test/MeshWeaver.MemexTemplate.Test/MemexTemplateGeneratorTest.cs b/test/MeshWeaver.MemexTemplate.Test/MemexTemplateGeneratorTest.cs index 420f368d99..581388d1c8 100644 --- a/test/MeshWeaver.MemexTemplate.Test/MemexTemplateGeneratorTest.cs +++ b/test/MeshWeaver.MemexTemplate.Test/MemexTemplateGeneratorTest.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.Net.Http; using System.Threading.Tasks; using System.Xml.Linq; using Xunit; @@ -30,8 +31,37 @@ public void Dispose() private string GeneratorScript => Path.Combine(RepoRoot, "tools", "generate-memex-template.cs"); + /// + /// Loud gate for the hidden network dependency (#679): dotnet run script.cs + /// implicitly RESTORES the file-based app, so an unreachable nuget.org fails every + /// test here with an NU1900 that points at NuGet internals — a phantom flake someone + /// then triages. Skipping with the cause on the result is honest. (Deliberately + /// duplicated from NuGetAssemblyResolverTest — this project is standalone, with no + /// shared test-utility reference to host a common helper.) + /// + private static async Task SkipUnlessNuGetOrgReachable() + { + using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(10) }; + try + { + using var response = await http.GetAsync("https://api.nuget.org/v3/index.json", + TestContext.Current.CancellationToken); + Assert.SkipUnless(response.IsSuccessStatusCode, + $"api.nuget.org service index answered {(int)response.StatusCode} — feed unhealthy; " + + "the generator's implicit restore needs the live feed (#679)"); + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) + { + Assert.Skip($"api.nuget.org unreachable ({ex.GetType().Name}: {ex.Message}) — the " + + "generator's implicit `dotnet run` restore needs the live feed; a network outage " + + "presents as a SKIP, not a phantom test failure (#679)"); + } + } + private async Task RunGenerator(string version = "0.0.0-test") { + await SkipUnlessNuGetOrgReachable(); + var psi = new ProcessStartInfo("dotnet", $"run \"{GeneratorScript}\" -- {version} \"{RepoRoot}\" \"{_outputPath}\"") { WorkingDirectory = RepoRoot, @@ -44,6 +74,19 @@ private async Task RunGenerator(string version = "0.0.0-test") var stderr = await process.StandardError.ReadToEndAsync(); await process.WaitForExitAsync(); + // The feed can die BETWEEN the probe above and the restore. NU1900 / "unable to + // load the service index" name exactly that condition and nothing the generator + // itself can cause — so it is the outage skip, not a masked generator defect. + if (process.ExitCode != 0) + { + var combined = stdout + stderr; + Assert.SkipWhen( + combined.Contains("NU1900") || + combined.Contains("Unable to load the service index", StringComparison.OrdinalIgnoreCase), + "nuget.org became unreachable during the generator's implicit restore — network " + + $"outage, not a generator defect (#679).\nstdout: {stdout}\nstderr: {stderr}"); + } + process.ExitCode.Should().Be(0, $"generator failed.\nstdout: {stdout}\nstderr: {stderr}"); _lastStdout = stdout; _lastStderr = stderr;