From af90ed713db2f31009c96de54d685c8f28dde3da Mon Sep 17 00:00:00 2001 From: Steve Pfister Date: Wed, 5 Aug 2026 14:04:15 -0400 Subject: [PATCH 01/10] Add diagnostics for Process test hangs Focus and serialize Windows ProcessStartInfo tests, log startup and test ordering, and capture a WER dump before the Helix timeout. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 54c890f9-b409-4b7d-b810-3701e125fd96 --- .../tests/AssemblyInfo.cs | 1 + .../tests/ProcessStartInfoTests.cs | 6 + .../tests/ProcessTestHangDiagnostics.cs | 111 ++++++++++++++++++ .../System.Diagnostics.Process.Tests.csproj | 6 + 4 files changed, 124 insertions(+) create mode 100644 src/libraries/System.Diagnostics.Process/tests/ProcessTestHangDiagnostics.cs diff --git a/src/libraries/System.Diagnostics.Process/tests/AssemblyInfo.cs b/src/libraries/System.Diagnostics.Process/tests/AssemblyInfo.cs index 80471364814581..607f83dab3569b 100644 --- a/src/libraries/System.Diagnostics.Process/tests/AssemblyInfo.cs +++ b/src/libraries/System.Diagnostics.Process/tests/AssemblyInfo.cs @@ -6,5 +6,6 @@ // Process tests can conflict with each other, as they modify ambient state // like the console code page and environment variables [assembly: CollectionBehavior(CollectionBehavior.CollectionPerAssembly)] +[assembly: System.Diagnostics.Tests.ProcessTestHangDiagnosticsAttribute] [assembly: SkipOnPlatform(TestPlatforms.Browser, "System.Diagnostics.Process is not supported on Browser.")] diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs index ba1eb8445aa33c..c23a708278c42d 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs @@ -1238,6 +1238,7 @@ private static string GetAssociationDetails() [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsWindowsNanoServer))] public void ShellExecute_Nano_Fails_Start() { + ProcessTestHangDiagnostics.Log("ShellExecute_Nano_Fails_Start started."); string tempFile = GetTestFilePath() + ".txt"; File.Create(tempFile).Dispose(); @@ -1250,7 +1251,9 @@ public void ShellExecute_Nano_Fails_Start() // Nano does not support either the STA apartment or ShellExecute. // Since we try to start an STA thread for ShellExecute, we hit a ThreadStartException // before we get to the PlatformNotSupportedException. + ProcessTestHangDiagnostics.Log("ShellExecute_Nano_Fails_Start calling Process.Start."); Assert.Throws(() => Process.Start(info)); + ProcessTestHangDiagnostics.Log("ShellExecute_Nano_Fails_Start completed Process.Start."); } public static TheoryData UseShellExecute @@ -1359,6 +1362,7 @@ public void InitializeWithArgumentList_ThrowsArgumentNullException() [ActiveIssue("https://github.com/dotnet/runtime/issues/34685", TestRuntimes.Mono)] public void StartInfo_NotepadWithContent_withArgumentList(bool useShellExecute) { + ProcessTestHangDiagnostics.Log($"StartInfo_NotepadWithContent_withArgumentList started; UseShellExecute={useShellExecute}."); string tempFile = GetTestFilePath() + ".txt"; File.WriteAllText(tempFile, $"StartInfo_NotepadWithContent({useShellExecute})"); @@ -1372,8 +1376,10 @@ public void StartInfo_NotepadWithContent_withArgumentList(bool useShellExecute) info.ArgumentList.Add(tempFile); + ProcessTestHangDiagnostics.Log($"StartInfo_NotepadWithContent_withArgumentList calling Process.Start; UseShellExecute={useShellExecute}."); using (var process = Process.Start(info)) { + ProcessTestHangDiagnostics.Log($"StartInfo_NotepadWithContent_withArgumentList completed Process.Start; UseShellExecute={useShellExecute}; ProcessId={process?.Id}."); Assert.True(process != null, $"Could not start {info.FileName} {info.Arguments} UseShellExecute={info.UseShellExecute}"); try diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessTestHangDiagnostics.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessTestHangDiagnostics.cs new file mode 100644 index 00000000000000..11df9e018cc10e --- /dev/null +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessTestHangDiagnostics.cs @@ -0,0 +1,111 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.IO; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Security; +using System.Text; +using System.Threading; +using Microsoft.Win32; +using Xunit.Sdk; + +namespace System.Diagnostics.Tests +{ + internal sealed class ProcessTestHangDiagnosticsAttribute : BeforeAfterTestAttribute + { + public override void Before(MethodInfo methodUnderTest) + { + ProcessTestHangDiagnostics.Log($"Starting {methodUnderTest.DeclaringType?.FullName}.{methodUnderTest.Name}."); + } + + public override void After(MethodInfo methodUnderTest) + { + ProcessTestHangDiagnostics.Log($"Finished {methodUnderTest.DeclaringType?.FullName}.{methodUnderTest.Name}."); + } + } + + internal static class ProcessTestHangDiagnostics + { +#if TargetsWindows + private const string InstallationTypeKey = @"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion"; + private static readonly TimeSpan WatchdogTimeout = TimeSpan.FromMinutes(3); + private static readonly TextWriter s_log = TextWriter.Synchronized( + new StreamWriter(Console.OpenStandardError(), Encoding.UTF8, bufferSize: 1024, leaveOpen: true) { AutoFlush = true }); + + [ModuleInitializer] + internal static void Initialize() + { + Log($"ProcessPath={Environment.ProcessPath}; OSVersion={Environment.OSVersion.Version}; Framework={RuntimeInformation.FrameworkDescription}"); + ConfigureWindowsErrorReporting(); + + var watchdog = new Thread(Watchdog) + { + IsBackground = true, + Name = "Process tests hang watchdog" + }; + watchdog.Start(); + + Log("Reading Windows InstallationType."); + object? installationType = Registry.GetValue(InstallationTypeKey, "InstallationType", defaultValue: null); + Log($"InstallationType={installationType ?? ""}"); + + Log("Evaluating PlatformDetection.IsWindowsNanoServer and IsWindowsServerCore."); + bool isWindowsNanoServer = PlatformDetection.IsWindowsNanoServer; + bool isWindowsServerCore = PlatformDetection.IsWindowsServerCore; + Log($"IsWindowsNanoServer={isWindowsNanoServer}; IsWindowsServerCore={isWindowsServerCore}"); + } + + internal static void Log(string message) + { + s_log.WriteLine($"[Process test hang diagnostics] {message}"); + } + + private static void ConfigureWindowsErrorReporting() + { + string? dumpFolder = Environment.GetEnvironmentVariable("HELIX_DUMP_FOLDER"); + string? processPath = Environment.ProcessPath; + if (string.IsNullOrEmpty(dumpFolder) || string.IsNullOrEmpty(processPath)) + { + Log($"WER LocalDumps not configured; HELIX_DUMP_FOLDER={dumpFolder ?? ""}."); + return; + } + + string executableName = Path.GetFileName(processPath); + string keyPath = $@"SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps\{executableName}"; + + try + { + using RegistryKey? key = Registry.LocalMachine.CreateSubKey(keyPath); + if (key is null) + { + Log($"Unable to create WER LocalDumps key HKLM\\{keyPath}."); + return; + } + + key.SetValue("DumpCount", 2, RegistryValueKind.DWord); + key.SetValue("DumpFolder", dumpFolder, RegistryValueKind.ExpandString); + key.SetValue("DumpType", 2, RegistryValueKind.DWord); + Log($"WER LocalDumps configured for {executableName} in {dumpFolder}."); + } + catch (Exception e) when (e is IOException or SecurityException or UnauthorizedAccessException) + { + Log($"WER LocalDumps configuration failed: {e}"); + } + } + + private static void Watchdog() + { + Thread.Sleep(WatchdogTimeout); + const string message = "System.Diagnostics.Process.Tests exceeded the diagnostic watchdog timeout."; + Log(message); + Environment.FailFast(message); + } +#else + internal static void Log(string message) + { + } +#endif + } +} diff --git a/src/libraries/System.Diagnostics.Process/tests/System.Diagnostics.Process.Tests.csproj b/src/libraries/System.Diagnostics.Process/tests/System.Diagnostics.Process.Tests.csproj index 59d512bf16abde..05ea8d989c23b1 100644 --- a/src/libraries/System.Diagnostics.Process/tests/System.Diagnostics.Process.Tests.csproj +++ b/src/libraries/System.Diagnostics.Process/tests/System.Diagnostics.Process.Tests.csproj @@ -11,6 +11,11 @@ $([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) $(DefineConstants);TargetsWindows + + true + $(XUnitOptions) -class System.Diagnostics.Tests.ProcessStartInfoTests -parallel none + true + @@ -33,6 +38,7 @@ + From 83813f663e8152a54175db14dc9dc4bfc13be573 Mon Sep 17 00:00:00 2001 From: Steve Pfister Date: Thu, 6 Aug 2026 10:30:40 -0400 Subject: [PATCH 02/10] Preserve Process hang diagnostics in Helix Write BOM-less UTF-8 console output and place WER dumps in the Helix work-item upload directory. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 54c890f9-b409-4b7d-b810-3701e125fd96 --- .../tests/ProcessTestHangDiagnostics.cs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessTestHangDiagnostics.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessTestHangDiagnostics.cs index 11df9e018cc10e..b8750f61441d6b 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessTestHangDiagnostics.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessTestHangDiagnostics.cs @@ -32,7 +32,7 @@ internal static class ProcessTestHangDiagnostics private const string InstallationTypeKey = @"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion"; private static readonly TimeSpan WatchdogTimeout = TimeSpan.FromMinutes(3); private static readonly TextWriter s_log = TextWriter.Synchronized( - new StreamWriter(Console.OpenStandardError(), Encoding.UTF8, bufferSize: 1024, leaveOpen: true) { AutoFlush = true }); + new StreamWriter(Console.OpenStandardError(), new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), bufferSize: 1024, leaveOpen: true) { AutoFlush = true }); [ModuleInitializer] internal static void Initialize() @@ -65,10 +65,12 @@ internal static void Log(string message) private static void ConfigureWindowsErrorReporting() { string? dumpFolder = Environment.GetEnvironmentVariable("HELIX_DUMP_FOLDER"); + string? uploadFolder = Environment.GetEnvironmentVariable("HELIX_WORKITEM_UPLOAD_ROOT"); + string? werDumpFolder = uploadFolder ?? dumpFolder; string? processPath = Environment.ProcessPath; - if (string.IsNullOrEmpty(dumpFolder) || string.IsNullOrEmpty(processPath)) + if (string.IsNullOrEmpty(werDumpFolder) || string.IsNullOrEmpty(processPath)) { - Log($"WER LocalDumps not configured; HELIX_DUMP_FOLDER={dumpFolder ?? ""}."); + Log($"WER LocalDumps not configured; HELIX_WORKITEM_UPLOAD_ROOT={uploadFolder ?? ""}; HELIX_DUMP_FOLDER={dumpFolder ?? ""}."); return; } @@ -85,9 +87,9 @@ private static void ConfigureWindowsErrorReporting() } key.SetValue("DumpCount", 2, RegistryValueKind.DWord); - key.SetValue("DumpFolder", dumpFolder, RegistryValueKind.ExpandString); + key.SetValue("DumpFolder", werDumpFolder, RegistryValueKind.ExpandString); key.SetValue("DumpType", 2, RegistryValueKind.DWord); - Log($"WER LocalDumps configured for {executableName} in {dumpFolder}."); + Log($"WER LocalDumps configured for {executableName} in {werDumpFolder}."); } catch (Exception e) when (e is IOException or SecurityException or UnauthorizedAccessException) { From 38282ce8f58f763dd8daa4ce6a2903647fab5a33 Mon Sep 17 00:00:00 2001 From: Steve Pfister Date: Thu, 6 Aug 2026 16:00:57 -0400 Subject: [PATCH 03/10] Use malformed PE for bad executable test Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../tests/ProcessStartInfoTests.cs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs index c23a708278c42d..18f569dcc840c2 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs @@ -1298,7 +1298,20 @@ public void StartInfo_BadVerb(bool useShellExecute) public void StartInfo_BadExe(bool useShellExecute) { string tempFile = GetTestFilePath() + ".exe"; - File.Create(tempFile).Dispose(); + const int MalformedExecutableSize = 512; + const int PeHeaderOffset = 0x80; + const int PeOptionalHeaderSizeOffset = PeHeaderOffset + 20; + const byte PeOptionalHeaderSize = 0xF0; + + // A truncated PE avoids special shell handling for an empty executable while still producing ERROR_BAD_EXE_FORMAT. + byte[] malformedExecutable = new byte[MalformedExecutableSize]; + malformedExecutable[0] = (byte)'M'; + malformedExecutable[1] = (byte)'Z'; + malformedExecutable[0x3C] = PeHeaderOffset; + malformedExecutable[PeHeaderOffset] = (byte)'P'; + malformedExecutable[PeHeaderOffset + 1] = (byte)'E'; + malformedExecutable[PeOptionalHeaderSizeOffset] = PeOptionalHeaderSize; + File.WriteAllBytes(tempFile, malformedExecutable); ProcessStartInfo info = new ProcessStartInfo { From 0cae6d674170fe18705e72517b0a74f81602e74f Mon Sep 17 00:00:00 2001 From: Steve Pfister Date: Thu, 6 Aug 2026 19:57:24 -0400 Subject: [PATCH 04/10] Use system DLL for bad executable test Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6a078136-b226-4ce5-94ff-1732ebcdfe79 --- .../tests/ProcessStartInfoTests.cs | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs index 18f569dcc840c2..c5f9053c9323ef 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs @@ -1298,20 +1298,8 @@ public void StartInfo_BadVerb(bool useShellExecute) public void StartInfo_BadExe(bool useShellExecute) { string tempFile = GetTestFilePath() + ".exe"; - const int MalformedExecutableSize = 512; - const int PeHeaderOffset = 0x80; - const int PeOptionalHeaderSizeOffset = PeHeaderOffset + 20; - const byte PeOptionalHeaderSize = 0xF0; - - // A truncated PE avoids special shell handling for an empty executable while still producing ERROR_BAD_EXE_FORMAT. - byte[] malformedExecutable = new byte[MalformedExecutableSize]; - malformedExecutable[0] = (byte)'M'; - malformedExecutable[1] = (byte)'Z'; - malformedExecutable[0x3C] = PeHeaderOffset; - malformedExecutable[PeHeaderOffset] = (byte)'P'; - malformedExecutable[PeHeaderOffset + 1] = (byte)'E'; - malformedExecutable[PeOptionalHeaderSizeOffset] = PeOptionalHeaderSize; - File.WriteAllBytes(tempFile, malformedExecutable); + // A DLL is a valid PE that cannot be executed, avoiding malformed-image shell recovery paths. + File.Copy(Path.Combine(Environment.SystemDirectory, "kernel32.dll"), tempFile); ProcessStartInfo info = new ProcessStartInfo { From 3116b8cc1d90c3cc37a30f28864fb6b23edbdc8b Mon Sep 17 00:00:00 2001 From: Steve Pfister Date: Sat, 8 Aug 2026 15:34:46 -0400 Subject: [PATCH 05/10] Use patched host for bad executable test Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6a078136-b226-4ce5-94ff-1732ebcdfe79 --- .../tests/ProcessStartInfoTests.cs | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs index c5f9053c9323ef..76cb2872b3bcf9 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs @@ -1297,9 +1297,39 @@ public void StartInfo_BadVerb(bool useShellExecute) [PlatformSpecific(TestPlatforms.Windows)] public void StartInfo_BadExe(bool useShellExecute) { + const ushort DosSignature = 0x5A4D; + const int PeHeaderOffsetPosition = 0x3C; + const uint PeSignature = 0x00004550; + const int CoffHeaderSize = 20; + const int CoffHeaderCharacteristicsOffset = 18; + const ushort ImageFileDll = 0x2000; + string tempFile = GetTestFilePath() + ".exe"; - // A DLL is a valid PE that cannot be executed, avoiding malformed-image shell recovery paths. - File.Copy(Path.Combine(Environment.SystemDirectory, "kernel32.dll"), tempFile); + string? processPath = Environment.ProcessPath; + Assert.NotNull(processPath); + File.Copy(processPath, tempFile); + + // Mark a copy of the current host as a DLL so it stays a valid PE but cannot be executed. + using (FileStream peStream = File.Open(tempFile, FileMode.Open, FileAccess.ReadWrite, FileShare.None)) + using (BinaryReader reader = new BinaryReader(peStream, Encoding.UTF8, leaveOpen: true)) + using (BinaryWriter writer = new BinaryWriter(peStream, Encoding.UTF8, leaveOpen: true)) + { + Assert.True(peStream.Length >= PeHeaderOffsetPosition + sizeof(int)); + Assert.Equal(DosSignature, reader.ReadUInt16()); + + peStream.Position = PeHeaderOffsetPosition; + int peHeaderOffset = reader.ReadInt32(); + Assert.InRange((long)peHeaderOffset, PeHeaderOffsetPosition + sizeof(int), peStream.Length - sizeof(uint) - CoffHeaderSize); + + peStream.Position = peHeaderOffset; + Assert.Equal(PeSignature, reader.ReadUInt32()); + + peStream.Position = peHeaderOffset + sizeof(uint) + CoffHeaderCharacteristicsOffset; + ushort characteristics = reader.ReadUInt16(); + Assert.Equal(0, characteristics & ImageFileDll); + peStream.Position -= sizeof(ushort); + writer.Write((ushort)(characteristics | ImageFileDll)); + } ProcessStartInfo info = new ProcessStartInfo { From 9476cb30fc277e849560e6e3cf3c7ac8f118f6c1 Mon Sep 17 00:00:00 2001 From: Steve Pfister Date: Sun, 9 Aug 2026 10:56:55 -0400 Subject: [PATCH 06/10] Skip hanging ShellExecute test on Server Core 2019 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6a078136-b226-4ce5-94ff-1732ebcdfe79 --- .../tests/AssemblyInfo.cs | 1 - .../tests/ProcessStartInfoTests.cs | 42 +------ .../tests/ProcessTestHangDiagnostics.cs | 113 ------------------ .../System.Diagnostics.Process.Tests.csproj | 6 - 4 files changed, 5 insertions(+), 157 deletions(-) delete mode 100644 src/libraries/System.Diagnostics.Process/tests/ProcessTestHangDiagnostics.cs diff --git a/src/libraries/System.Diagnostics.Process/tests/AssemblyInfo.cs b/src/libraries/System.Diagnostics.Process/tests/AssemblyInfo.cs index 607f83dab3569b..80471364814581 100644 --- a/src/libraries/System.Diagnostics.Process/tests/AssemblyInfo.cs +++ b/src/libraries/System.Diagnostics.Process/tests/AssemblyInfo.cs @@ -6,6 +6,5 @@ // Process tests can conflict with each other, as they modify ambient state // like the console code page and environment variables [assembly: CollectionBehavior(CollectionBehavior.CollectionPerAssembly)] -[assembly: System.Diagnostics.Tests.ProcessTestHangDiagnosticsAttribute] [assembly: SkipOnPlatform(TestPlatforms.Browser, "System.Diagnostics.Process is not supported on Browser.")] diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs index 76cb2872b3bcf9..aed643eabd63b1 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs @@ -1238,7 +1238,6 @@ private static string GetAssociationDetails() [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsWindowsNanoServer))] public void ShellExecute_Nano_Fails_Start() { - ProcessTestHangDiagnostics.Log("ShellExecute_Nano_Fails_Start started."); string tempFile = GetTestFilePath() + ".txt"; File.Create(tempFile).Dispose(); @@ -1251,9 +1250,7 @@ public void ShellExecute_Nano_Fails_Start() // Nano does not support either the STA apartment or ShellExecute. // Since we try to start an STA thread for ShellExecute, we hit a ThreadStartException // before we get to the PlatformNotSupportedException. - ProcessTestHangDiagnostics.Log("ShellExecute_Nano_Fails_Start calling Process.Start."); Assert.Throws(() => Process.Start(info)); - ProcessTestHangDiagnostics.Log("ShellExecute_Nano_Fails_Start completed Process.Start."); } public static TheoryData UseShellExecute @@ -1297,39 +1294,13 @@ public void StartInfo_BadVerb(bool useShellExecute) [PlatformSpecific(TestPlatforms.Windows)] public void StartInfo_BadExe(bool useShellExecute) { - const ushort DosSignature = 0x5A4D; - const int PeHeaderOffsetPosition = 0x3C; - const uint PeSignature = 0x00004550; - const int CoffHeaderSize = 20; - const int CoffHeaderCharacteristicsOffset = 18; - const ushort ImageFileDll = 0x2000; + if (useShellExecute && PlatformDetection.IsWindowsServer2019 && PlatformDetection.IsWindowsServerCore) + { + throw new SkipTestException("ShellExecuteEx hangs while activating Windows StateRepository on Server Core 2019."); + } string tempFile = GetTestFilePath() + ".exe"; - string? processPath = Environment.ProcessPath; - Assert.NotNull(processPath); - File.Copy(processPath, tempFile); - - // Mark a copy of the current host as a DLL so it stays a valid PE but cannot be executed. - using (FileStream peStream = File.Open(tempFile, FileMode.Open, FileAccess.ReadWrite, FileShare.None)) - using (BinaryReader reader = new BinaryReader(peStream, Encoding.UTF8, leaveOpen: true)) - using (BinaryWriter writer = new BinaryWriter(peStream, Encoding.UTF8, leaveOpen: true)) - { - Assert.True(peStream.Length >= PeHeaderOffsetPosition + sizeof(int)); - Assert.Equal(DosSignature, reader.ReadUInt16()); - - peStream.Position = PeHeaderOffsetPosition; - int peHeaderOffset = reader.ReadInt32(); - Assert.InRange((long)peHeaderOffset, PeHeaderOffsetPosition + sizeof(int), peStream.Length - sizeof(uint) - CoffHeaderSize); - - peStream.Position = peHeaderOffset; - Assert.Equal(PeSignature, reader.ReadUInt32()); - - peStream.Position = peHeaderOffset + sizeof(uint) + CoffHeaderCharacteristicsOffset; - ushort characteristics = reader.ReadUInt16(); - Assert.Equal(0, characteristics & ImageFileDll); - peStream.Position -= sizeof(ushort); - writer.Write((ushort)(characteristics | ImageFileDll)); - } + File.Create(tempFile).Dispose(); ProcessStartInfo info = new ProcessStartInfo { @@ -1393,7 +1364,6 @@ public void InitializeWithArgumentList_ThrowsArgumentNullException() [ActiveIssue("https://github.com/dotnet/runtime/issues/34685", TestRuntimes.Mono)] public void StartInfo_NotepadWithContent_withArgumentList(bool useShellExecute) { - ProcessTestHangDiagnostics.Log($"StartInfo_NotepadWithContent_withArgumentList started; UseShellExecute={useShellExecute}."); string tempFile = GetTestFilePath() + ".txt"; File.WriteAllText(tempFile, $"StartInfo_NotepadWithContent({useShellExecute})"); @@ -1407,10 +1377,8 @@ public void StartInfo_NotepadWithContent_withArgumentList(bool useShellExecute) info.ArgumentList.Add(tempFile); - ProcessTestHangDiagnostics.Log($"StartInfo_NotepadWithContent_withArgumentList calling Process.Start; UseShellExecute={useShellExecute}."); using (var process = Process.Start(info)) { - ProcessTestHangDiagnostics.Log($"StartInfo_NotepadWithContent_withArgumentList completed Process.Start; UseShellExecute={useShellExecute}; ProcessId={process?.Id}."); Assert.True(process != null, $"Could not start {info.FileName} {info.Arguments} UseShellExecute={info.UseShellExecute}"); try diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessTestHangDiagnostics.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessTestHangDiagnostics.cs deleted file mode 100644 index b8750f61441d6b..00000000000000 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessTestHangDiagnostics.cs +++ /dev/null @@ -1,113 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.IO; -using System.Reflection; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Security; -using System.Text; -using System.Threading; -using Microsoft.Win32; -using Xunit.Sdk; - -namespace System.Diagnostics.Tests -{ - internal sealed class ProcessTestHangDiagnosticsAttribute : BeforeAfterTestAttribute - { - public override void Before(MethodInfo methodUnderTest) - { - ProcessTestHangDiagnostics.Log($"Starting {methodUnderTest.DeclaringType?.FullName}.{methodUnderTest.Name}."); - } - - public override void After(MethodInfo methodUnderTest) - { - ProcessTestHangDiagnostics.Log($"Finished {methodUnderTest.DeclaringType?.FullName}.{methodUnderTest.Name}."); - } - } - - internal static class ProcessTestHangDiagnostics - { -#if TargetsWindows - private const string InstallationTypeKey = @"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion"; - private static readonly TimeSpan WatchdogTimeout = TimeSpan.FromMinutes(3); - private static readonly TextWriter s_log = TextWriter.Synchronized( - new StreamWriter(Console.OpenStandardError(), new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), bufferSize: 1024, leaveOpen: true) { AutoFlush = true }); - - [ModuleInitializer] - internal static void Initialize() - { - Log($"ProcessPath={Environment.ProcessPath}; OSVersion={Environment.OSVersion.Version}; Framework={RuntimeInformation.FrameworkDescription}"); - ConfigureWindowsErrorReporting(); - - var watchdog = new Thread(Watchdog) - { - IsBackground = true, - Name = "Process tests hang watchdog" - }; - watchdog.Start(); - - Log("Reading Windows InstallationType."); - object? installationType = Registry.GetValue(InstallationTypeKey, "InstallationType", defaultValue: null); - Log($"InstallationType={installationType ?? ""}"); - - Log("Evaluating PlatformDetection.IsWindowsNanoServer and IsWindowsServerCore."); - bool isWindowsNanoServer = PlatformDetection.IsWindowsNanoServer; - bool isWindowsServerCore = PlatformDetection.IsWindowsServerCore; - Log($"IsWindowsNanoServer={isWindowsNanoServer}; IsWindowsServerCore={isWindowsServerCore}"); - } - - internal static void Log(string message) - { - s_log.WriteLine($"[Process test hang diagnostics] {message}"); - } - - private static void ConfigureWindowsErrorReporting() - { - string? dumpFolder = Environment.GetEnvironmentVariable("HELIX_DUMP_FOLDER"); - string? uploadFolder = Environment.GetEnvironmentVariable("HELIX_WORKITEM_UPLOAD_ROOT"); - string? werDumpFolder = uploadFolder ?? dumpFolder; - string? processPath = Environment.ProcessPath; - if (string.IsNullOrEmpty(werDumpFolder) || string.IsNullOrEmpty(processPath)) - { - Log($"WER LocalDumps not configured; HELIX_WORKITEM_UPLOAD_ROOT={uploadFolder ?? ""}; HELIX_DUMP_FOLDER={dumpFolder ?? ""}."); - return; - } - - string executableName = Path.GetFileName(processPath); - string keyPath = $@"SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps\{executableName}"; - - try - { - using RegistryKey? key = Registry.LocalMachine.CreateSubKey(keyPath); - if (key is null) - { - Log($"Unable to create WER LocalDumps key HKLM\\{keyPath}."); - return; - } - - key.SetValue("DumpCount", 2, RegistryValueKind.DWord); - key.SetValue("DumpFolder", werDumpFolder, RegistryValueKind.ExpandString); - key.SetValue("DumpType", 2, RegistryValueKind.DWord); - Log($"WER LocalDumps configured for {executableName} in {werDumpFolder}."); - } - catch (Exception e) when (e is IOException or SecurityException or UnauthorizedAccessException) - { - Log($"WER LocalDumps configuration failed: {e}"); - } - } - - private static void Watchdog() - { - Thread.Sleep(WatchdogTimeout); - const string message = "System.Diagnostics.Process.Tests exceeded the diagnostic watchdog timeout."; - Log(message); - Environment.FailFast(message); - } -#else - internal static void Log(string message) - { - } -#endif - } -} diff --git a/src/libraries/System.Diagnostics.Process/tests/System.Diagnostics.Process.Tests.csproj b/src/libraries/System.Diagnostics.Process/tests/System.Diagnostics.Process.Tests.csproj index 05ea8d989c23b1..59d512bf16abde 100644 --- a/src/libraries/System.Diagnostics.Process/tests/System.Diagnostics.Process.Tests.csproj +++ b/src/libraries/System.Diagnostics.Process/tests/System.Diagnostics.Process.Tests.csproj @@ -11,11 +11,6 @@ $([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) $(DefineConstants);TargetsWindows - - true - $(XUnitOptions) -class System.Diagnostics.Tests.ProcessStartInfoTests -parallel none - true - @@ -38,7 +33,6 @@ - From c088ec8b349b843028fef5b3f1287fa9b8432ea2 Mon Sep 17 00:00:00 2001 From: Steve Pfister Date: Fri, 2 Oct 2026 12:57:05 -0400 Subject: [PATCH 07/10] Add temporary diagnostics for Windows x86 Process test hangs Restore bad-executable coverage, enable the existing verbose xUnit progress reporter, and capture a full test-host dump before the Helix timeout using the matching Windows createdump tool. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../System.Diagnostics.Process/README.md | 18 ++++ .../tests/ProcessStartInfoTests.cs | 5 - .../tests/ProcessTestHangDiagnostics.cs | 100 ++++++++++++++++++ .../System.Diagnostics.Process.Tests.csproj | 4 + 4 files changed, 122 insertions(+), 5 deletions(-) create mode 100644 src/libraries/System.Diagnostics.Process/tests/ProcessTestHangDiagnostics.cs diff --git a/src/libraries/System.Diagnostics.Process/README.md b/src/libraries/System.Diagnostics.Process/README.md index c1c29f88d6dc86..7322ddfb8c9c3c 100644 --- a/src/libraries/System.Diagnostics.Process/README.md +++ b/src/libraries/System.Diagnostics.Process/README.md @@ -11,3 +11,21 @@ See the [Help Wanted](https://github.com/dotnet/runtime/issues?q=is%3Aopen+is%3A ## Deployment `System.Diagnostics.Process` is included in the shared framework. The package does not need to be installed into any project compatible with .NET Standard 2.0. + +## Temporary hang diagnostics + +This investigation branch enables xUnit's verbose start/finish reporting on Windows, including +theory arguments, without changing test selection or parallelism. These messages are independent +of `XUNIT_HIDE_PASSING_OUTPUT_DIAGNOSTICS`. + +For Windows x86 CoreCLR Helix runs, an eight-minute background watchdog in the xUnit test host +launches that runtime's `createdump.exe --full`. On Windows, this tool dumps its parent process; +it is not launched through a shell and no PID lookup or WER registry configuration is needed. +RemoteExecutor children do not arm the watchdog, and normal runner exit ends the background thread. +Local runs without `HELIX_WORKITEM_UPLOAD_ROOT` do not arm it. + +The dump is written directly to `HELIX_WORKITEM_UPLOAD_ROOT` for Helix artifact collection. +Dump capture has a two-minute budget, followed by at most 30 seconds to terminate the dumper. +The test host then exits with code 124, including if dump capture fails, before Helix's +15-minute timeout. Capture failures are logged explicitly. These diagnostics are temporary +and are not a production fix or evidence that different timeout reports share a root cause. diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs index 39e43b79d3dcba..a187c49c733c4b 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs @@ -1316,11 +1316,6 @@ public void StartInfo_BadVerb(bool useShellExecute) [PlatformSpecific(TestPlatforms.Windows)] public void StartInfo_BadExe(bool useShellExecute) { - if (useShellExecute && PlatformDetection.IsWindowsServer2019 && PlatformDetection.IsWindowsServerCore) - { - throw new SkipTestException("ShellExecuteEx hangs while activating Windows StateRepository on Server Core 2019."); - } - string tempFile = GetTestFilePath() + ".exe"; File.Create(tempFile).Dispose(); diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessTestHangDiagnostics.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessTestHangDiagnostics.cs new file mode 100644 index 00000000000000..c7b7ee2ff7b4bf --- /dev/null +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessTestHangDiagnostics.cs @@ -0,0 +1,100 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.ComponentModel; +using System.IO; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading; + +namespace System.Diagnostics.Tests; + +internal static class ProcessTestHangDiagnostics +{ + private static readonly TimeSpan s_watchdogTimeout = TimeSpan.FromMinutes(8); + private static readonly TimeSpan s_dumpTimeout = TimeSpan.FromMinutes(2); + + [ModuleInitializer] + internal static void Initialize() + { + string? uploadRoot = Environment.GetEnvironmentVariable("HELIX_WORKITEM_UPLOAD_ROOT"); + if (string.IsNullOrEmpty(uploadRoot) || + !string.Equals(Path.GetFileName(Environment.GetCommandLineArgs()[0]), "xunit.console.dll", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + string dumpTool = Path.Combine(RuntimeEnvironment.GetRuntimeDirectory(), "createdump.exe"); + if (!File.Exists(dumpTool)) + { + throw new FileNotFoundException("The diagnostic watchdog requires the test runtime's Windows createdump.exe.", dumpTool); + } + + Directory.CreateDirectory(uploadRoot); + var watchdog = new Thread(() => Watchdog(dumpTool, uploadRoot, s_watchdogTimeout)) + { + // The watchdog belongs to the runner, not to RemoteExecutor children, and ends with the runner. + IsBackground = true, + Name = "Process tests hang watchdog" + }; + watchdog.Start(); + } + + private static void Watchdog(string dumpTool, string uploadRoot, TimeSpan watchdogTimeout) + { + // Bypass xUnit's per-test output capture, including passing-output suppression. + using var log = new StreamWriter(Console.OpenStandardError(), new UTF8Encoding(false), leaveOpen: true) + { + AutoFlush = true + }; + log.WriteLine($"[Process hang diagnostics] Armed for {watchdogTimeout}; PID={Environment.ProcessId}; architecture={RuntimeInformation.ProcessArchitecture}; OS={Environment.OSVersion}; createdump={dumpTool}; upload={uploadRoot}"); + + Thread.Sleep(watchdogTimeout); + string dumpPath = Path.Combine(uploadRoot, $"ProcessTests.{Environment.ProcessId}.dmp"); + log.WriteLine($"[Process hang diagnostics] Watchdog expired. Capturing full test-host dump to {dumpPath}."); + try + { + CaptureDump(dumpTool, dumpPath, s_dumpTimeout, log); + } + catch (Exception e) when (e is IOException or UnauthorizedAccessException or Win32Exception or InvalidOperationException) + { + log.WriteLine($"[Process hang diagnostics] Dump capture failed: {e}"); + } + + // Return a distinct failure before Helix's 900s kill, leaving time for artifact upload. + log.WriteLine("[Process hang diagnostics] Ending the timed-out test host with exit code 124."); + Environment.Exit(124); + } + + private static void CaptureDump(string dumpTool, string dumpPath, TimeSpan dumpTimeout, TextWriter log) + { + var startInfo = new ProcessStartInfo(dumpTool) + { + UseShellExecute = false + }; + startInfo.ArgumentList.Add("--full"); + startInfo.ArgumentList.Add("--name"); + startInfo.ArgumentList.Add(dumpPath); + + // Windows createdump targets its parent, so launch the matching runtime's tool directly from the test host. + using Process dumper = Process.Start(startInfo) ?? throw new InvalidOperationException("Could not start createdump."); + if (!dumper.WaitForExit((int)dumpTimeout.TotalMilliseconds)) + { + dumper.Kill(); + if (!dumper.WaitForExit(30_000)) + { + throw new InvalidOperationException($"createdump PID {dumper.Id} did not exit after termination."); + } + + throw new IOException($"createdump exceeded its {dumpTimeout} budget."); + } + + if (dumper.ExitCode != 0 || !File.Exists(dumpPath) || new FileInfo(dumpPath).Length == 0) + { + throw new IOException($"createdump exited with code {dumper.ExitCode} without a successful nonempty dump at {dumpPath}."); + } + + log.WriteLine($"[Process hang diagnostics] Full dump captured: {dumpPath} ({new FileInfo(dumpPath).Length} bytes)."); + } +} diff --git a/src/libraries/System.Diagnostics.Process/tests/System.Diagnostics.Process.Tests.csproj b/src/libraries/System.Diagnostics.Process/tests/System.Diagnostics.Process.Tests.csproj index 100ff174567dab..f83b32ce5d202d 100644 --- a/src/libraries/System.Diagnostics.Process/tests/System.Diagnostics.Process.Tests.csproj +++ b/src/libraries/System.Diagnostics.Process/tests/System.Diagnostics.Process.Tests.csproj @@ -10,7 +10,11 @@ $([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) $(DefineConstants);TargetsWindows + true + + + From 3f62e54002996709725f009431e6084c68f1276f Mon Sep 17 00:00:00 2001 From: Steve Pfister Date: Fri, 2 Oct 2026 13:08:20 -0400 Subject: [PATCH 08/10] Enable only the diagnostic PR's Windows x86 checked-libraries leg Opt PR 131907 into the existing build and Helix conditions without changing test scope. Exclude only its diagnostic runtime.yml edit from path triggers, leaving other PRs and genuine CoreCLR changes unaffected. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../common/evaluate-default-paths.yml | 18 ++++++++++++++++++ eng/pipelines/runtime.yml | 15 +++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/common/evaluate-default-paths.yml b/eng/pipelines/common/evaluate-default-paths.yml index dba1aaad202b60..4874c6b6ffcb1c 100644 --- a/eng/pipelines/common/evaluate-default-paths.yml +++ b/eng/pipelines/common/evaluate-default-paths.yml @@ -114,6 +114,8 @@ jobs: - ${{ parameters._const_paths._wasm_specific_only }} - ${{ parameters._const_paths._wasm_pipelines }} - ${{ parameters._const_paths._always_exclude }} + - ${{ if eq(variables['Build.SourceBranch'], 'refs/pull/131907/merge') }}: + - eng/pipelines/runtime.yml - ${{ parameters._const_paths._perf_pipeline_specific_only }} - subset: mono_excluding_wasm @@ -126,6 +128,8 @@ jobs: - ${{ parameters._const_paths._wasm_specific_only }} - ${{ parameters._const_paths._wasm_pipelines }} - ${{ parameters._const_paths._always_exclude }} + - ${{ if eq(variables['Build.SourceBranch'], 'refs/pull/131907/merge') }}: + - eng/pipelines/runtime.yml - ${{ parameters._const_paths._perf_pipeline_specific_only }} - eng/Version.Details.xml @@ -158,6 +162,8 @@ jobs: - ${{ parameters._const_paths._wasm_specific_only }} - ${{ parameters._const_paths._wasm_pipelines }} - ${{ parameters._const_paths._always_exclude }} + - ${{ if eq(variables['Build.SourceBranch'], 'refs/pull/131907/merge') }}: + - eng/pipelines/runtime.yml - ${{ parameters._const_paths._perf_pipeline_specific_only }} - subset: runtimetests @@ -198,6 +204,8 @@ jobs: - ${{ parameters._const_paths._wasm_specific_only }} - ${{ parameters._const_paths._wasm_pipelines }} - ${{ parameters._const_paths._always_exclude }} + - ${{ if eq(variables['Build.SourceBranch'], 'refs/pull/131907/merge') }}: + - eng/pipelines/runtime.yml - ${{ parameters._const_paths._perf_pipeline_specific_only }} # We have limited Apple Silicon testing capacity @@ -267,6 +275,8 @@ jobs: - ${{ parameters._const_paths._wasm_pipelines }} exclude: - ${{ parameters._const_paths._always_exclude }} + - ${{ if eq(variables['Build.SourceBranch'], 'refs/pull/131907/merge') }}: + - eng/pipelines/runtime.yml - ${{ parameters._const_paths._perf_pipeline_specific_only }} # wasm/runtimetests mono need to be run @@ -289,6 +299,8 @@ jobs: - src/mono/wasm/Wasm.Build.Tests/* - ${{ parameters._const_paths._wasm_pipelines }} - ${{ parameters._const_paths._always_exclude }} + - ${{ if eq(variables['Build.SourceBranch'], 'refs/pull/131907/merge') }}: + - eng/pipelines/runtime.yml - ${{ parameters._const_paths._perf_pipeline_specific_only }} # wasm/runtimetests coreCLR need to be run @@ -302,6 +314,8 @@ jobs: - src/mono/* - ${{ parameters._const_paths._wasm_pipelines }} - ${{ parameters._const_paths._always_exclude }} + - ${{ if eq(variables['Build.SourceBranch'], 'refs/pull/131907/merge') }}: + - eng/pipelines/runtime.yml - ${{ parameters._const_paths._perf_pipeline_specific_only }} # Wasm-specific paths excluding Wasm.Build.Tests, WASI (build/runtime/testassets), and the debugger @@ -331,6 +345,8 @@ jobs: - src/mono/nuget/Microsoft.NET.Workload* - src/mono/nuget/Microsoft.NETCore.BrowserDebugHost.Transport/* - ${{ parameters._const_paths._always_exclude }} + - ${{ if eq(variables['Build.SourceBranch'], 'refs/pull/131907/merge') }}: + - eng/pipelines/runtime.yml - ${{ parameters._const_paths._perf_pipeline_specific_only }} - subset: wasm_chrome @@ -344,6 +360,8 @@ jobs: - ${{ parameters._const_paths._wasm_specific_only }} - ${{ parameters._const_paths._wasm_pipelines }} - ${{ parameters._const_paths._always_exclude }} + - ${{ if eq(variables['Build.SourceBranch'], 'refs/pull/131907/merge') }}: + - eng/pipelines/runtime.yml - ${{ parameters._const_paths._perf_pipeline_specific_only }} - eng/testing/tests.mobile.targets - src/mono/* diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 9fabe16d48fd80..9759b94d7e1d3e 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -557,14 +557,25 @@ extends: testRunNamePrefixSuffix: coreclr_checked testScope: innerloop useHelixMonitor: ${{ variables.enableHelixJobMonitor }} + # Temporary opt-in for this diagnostic PR's Windows x86 checked-libraries leg only. condition: >- or( eq(variables['coreclrContainsChange'], true), - eq(variables['isRollingBuild'], true)) + eq(variables['isRollingBuild'], true), + and( + eq(variables['Build.Reason'], 'PullRequest'), + eq(variables['Build.SourceBranch'], 'refs/pull/131907/merge'), + eq(variables['osGroup'], 'windows'), + eq(variables['archType'], 'x86'))) condition: >- or( eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), - eq(variables['isRollingBuild'], true)) + eq(variables['isRollingBuild'], true), + and( + eq(variables['Build.Reason'], 'PullRequest'), + eq(variables['Build.SourceBranch'], 'refs/pull/131907/merge'), + eq(variables['osGroup'], 'windows'), + eq(variables['archType'], 'x86'))) - template: /eng/pipelines/common/platform-matrix.yml parameters: From 377b58b548c929d6c1521872a3e44d585485e120 Mon Sep 17 00:00:00 2001 From: Steve Pfister Date: Fri, 2 Oct 2026 16:42:34 -0400 Subject: [PATCH 09/10] Make Process diagnostics capture a snapshot without ending tests Keep the eight-minute full-memory snapshot and original Helix deadline, log capture failures without changing test status, and reap an in-flight dump child when the host completes naturally. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../System.Diagnostics.Process/README.md | 12 +- .../tests/ProcessTestHangDiagnostics.cs | 113 +++++++++++++----- 2 files changed, 95 insertions(+), 30 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/README.md b/src/libraries/System.Diagnostics.Process/README.md index 7322ddfb8c9c3c..7d221563d97bd6 100644 --- a/src/libraries/System.Diagnostics.Process/README.md +++ b/src/libraries/System.Diagnostics.Process/README.md @@ -22,10 +22,16 @@ For Windows x86 CoreCLR Helix runs, an eight-minute background watchdog in the x launches that runtime's `createdump.exe --full`. On Windows, this tool dumps its parent process; it is not launched through a shell and no PID lookup or WER registry configuration is needed. RemoteExecutor children do not arm the watchdog, and normal runner exit ends the background thread. +An in-flight dump child is terminated and reaped if the runner exits during capture. Local runs without `HELIX_WORKITEM_UPLOAD_ROOT` do not arm it. The dump is written directly to `HELIX_WORKITEM_UPLOAD_ROOT` for Helix artifact collection. Dump capture has a two-minute budget, followed by at most 30 seconds to terminate the dumper. -The test host then exits with code 124, including if dump capture fails, before Helix's -15-minute timeout. Capture failures are logged explicitly. These diagnostics are temporary -and are not a production fix or evidence that different timeout reports share a root cause. +This is a snapshot only: successful capture and diagnostic failures do not terminate the test host +or change its exit status. Capture and cleanup failures are logged explicitly to standard error. +The full suite continues until ordinary completion or Helix's original 15-minute deadline, +allowing slow progress to be distinguished from a whole-work-item timeout. + +These diagnostics investigate the unexplained Process-suite timeout independently of earlier +investigations. They are temporary, not a production fix, and do not establish that different +timeout reports share a root cause. diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessTestHangDiagnostics.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessTestHangDiagnostics.cs index c7b7ee2ff7b4bf..0e6e131efd7b02 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessTestHangDiagnostics.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessTestHangDiagnostics.cs @@ -14,6 +14,7 @@ internal static class ProcessTestHangDiagnostics { private static readonly TimeSpan s_watchdogTimeout = TimeSpan.FromMinutes(8); private static readonly TimeSpan s_dumpTimeout = TimeSpan.FromMinutes(2); + private static readonly TimeSpan s_dumperExitTimeout = TimeSpan.FromSeconds(30); [ModuleInitializer] internal static void Initialize() @@ -26,13 +27,7 @@ internal static void Initialize() } string dumpTool = Path.Combine(RuntimeEnvironment.GetRuntimeDirectory(), "createdump.exe"); - if (!File.Exists(dumpTool)) - { - throw new FileNotFoundException("The diagnostic watchdog requires the test runtime's Windows createdump.exe.", dumpTool); - } - - Directory.CreateDirectory(uploadRoot); - var watchdog = new Thread(() => Watchdog(dumpTool, uploadRoot, s_watchdogTimeout)) + var watchdog = new Thread(() => Watchdog(dumpTool, uploadRoot, s_watchdogTimeout, s_dumpTimeout)) { // The watchdog belongs to the runner, not to RemoteExecutor children, and ends with the runner. IsBackground = true, @@ -41,30 +36,27 @@ internal static void Initialize() watchdog.Start(); } - private static void Watchdog(string dumpTool, string uploadRoot, TimeSpan watchdogTimeout) + private static void Watchdog(string dumpTool, string uploadRoot, TimeSpan watchdogTimeout, TimeSpan dumpTimeout) { // Bypass xUnit's per-test output capture, including passing-output suppression. - using var log = new StreamWriter(Console.OpenStandardError(), new UTF8Encoding(false), leaveOpen: true) + using TextWriter log = TextWriter.Synchronized(new StreamWriter(Console.OpenStandardError(), new UTF8Encoding(false), leaveOpen: true) { AutoFlush = true - }; + }); log.WriteLine($"[Process hang diagnostics] Armed for {watchdogTimeout}; PID={Environment.ProcessId}; architecture={RuntimeInformation.ProcessArchitecture}; OS={Environment.OSVersion}; createdump={dumpTool}; upload={uploadRoot}"); Thread.Sleep(watchdogTimeout); string dumpPath = Path.Combine(uploadRoot, $"ProcessTests.{Environment.ProcessId}.dmp"); - log.WriteLine($"[Process hang diagnostics] Watchdog expired. Capturing full test-host dump to {dumpPath}."); + log.WriteLine($"[Process hang diagnostics] Snapshot time reached. Capturing full test-host dump to {dumpPath} without terminating tests."); try { - CaptureDump(dumpTool, dumpPath, s_dumpTimeout, log); + Directory.CreateDirectory(uploadRoot); + CaptureDump(dumpTool, dumpPath, dumpTimeout, log); } - catch (Exception e) when (e is IOException or UnauthorizedAccessException or Win32Exception or InvalidOperationException) + catch (Exception e) when (e is IOException or UnauthorizedAccessException or Win32Exception or InvalidOperationException or OperationCanceledException) { log.WriteLine($"[Process hang diagnostics] Dump capture failed: {e}"); } - - // Return a distinct failure before Helix's 900s kill, leaving time for artifact upload. - log.WriteLine("[Process hang diagnostics] Ending the timed-out test host with exit code 124."); - Environment.Exit(124); } private static void CaptureDump(string dumpTool, string dumpPath, TimeSpan dumpTimeout, TextWriter log) @@ -77,24 +69,91 @@ private static void CaptureDump(string dumpTool, string dumpPath, TimeSpan dumpT startInfo.ArgumentList.Add("--name"); startInfo.ArgumentList.Add(dumpPath); - // Windows createdump targets its parent, so launch the matching runtime's tool directly from the test host. - using Process dumper = Process.Start(startInfo) ?? throw new InvalidOperationException("Could not start createdump."); - if (!dumper.WaitForExit((int)dumpTimeout.TotalMilliseconds)) + using var dumper = new Process { StartInfo = startInfo }; + var exitLock = new object(); + bool started = false; + bool finished = false; + bool hostExiting = false; + + AppDomain.CurrentDomain.ProcessExit += OnProcessExit; + try { - dumper.Kill(); - if (!dumper.WaitForExit(30_000)) + lock (exitLock) + { + if (hostExiting) + { + throw new OperationCanceledException("The test host exited before the snapshot could start."); + } + + // Windows createdump targets its parent, so start it directly in the test host. + started = dumper.Start(); + if (!started) + { + throw new InvalidOperationException("Could not start createdump."); + } + } + + log.WriteLine($"[Process hang diagnostics] createdump PID={dumper.Id} started."); + if (!dumper.WaitForExit((int)dumpTimeout.TotalMilliseconds)) + { + throw new IOException($"createdump exceeded its {dumpTimeout} budget."); + } + + if (dumper.ExitCode != 0 || !File.Exists(dumpPath) || new FileInfo(dumpPath).Length == 0) { - throw new InvalidOperationException($"createdump PID {dumper.Id} did not exit after termination."); + throw new IOException($"createdump exited with code {dumper.ExitCode} without a successful nonempty dump at {dumpPath}."); } - throw new IOException($"createdump exceeded its {dumpTimeout} budget."); + log.WriteLine($"[Process hang diagnostics] Full dump captured: {dumpPath} ({new FileInfo(dumpPath).Length} bytes). Test execution continues."); + } + finally + { + lock (exitLock) + { + finished = true; + AppDomain.CurrentDomain.ProcessExit -= OnProcessExit; + if (started) + { + StopDumper(dumper); + } + } } - if (dumper.ExitCode != 0 || !File.Exists(dumpPath) || new FileInfo(dumpPath).Length == 0) + void OnProcessExit(object? sender, EventArgs args) { - throw new IOException($"createdump exited with code {dumper.ExitCode} without a successful nonempty dump at {dumpPath}."); + lock (exitLock) + { + if (finished) + { + return; + } + + hostExiting = true; + if (started) + { + try + { + StopDumper(dumper); + } + catch (Exception e) when (e is Win32Exception or InvalidOperationException) + { + log.WriteLine($"[Process hang diagnostics] Dump child cleanup failed during test-host exit: {e}"); + } + } + } } + } - log.WriteLine($"[Process hang diagnostics] Full dump captured: {dumpPath} ({new FileInfo(dumpPath).Length} bytes)."); + private static void StopDumper(Process dumper) + { + if (!dumper.HasExited) + { + dumper.Kill(); + } + + if (!dumper.WaitForExit((int)s_dumperExitTimeout.TotalMilliseconds)) + { + throw new InvalidOperationException($"createdump PID {dumper.Id} did not exit after termination."); + } } } From 571ecdf5c73dbe14d84d7b31dd9843faa77ccf4e Mon Sep 17 00:00:00 2001 From: Steve Pfister Date: Fri, 2 Oct 2026 22:25:34 -0400 Subject: [PATCH 10/10] Partition Windows x86 Process tests into balanced Helix work items Generate six isolated console-runner archives using existing script and ZIP tasks, preserving all class/theory coverage and full local/static-runner semantics. Remove temporary dump instrumentation; retain the narrow PR CI opt-in for elapsed-time validation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../System.Diagnostics.Process/README.md | 37 ++-- .../tests/ProcessTestHangDiagnostics.cs | 159 ------------------ .../tests/ProcessTestShards.targets | 73 ++++++++ .../System.Diagnostics.Process.Tests.csproj | 5 +- 4 files changed, 88 insertions(+), 186 deletions(-) delete mode 100644 src/libraries/System.Diagnostics.Process/tests/ProcessTestHangDiagnostics.cs create mode 100644 src/libraries/System.Diagnostics.Process/tests/ProcessTestShards.targets diff --git a/src/libraries/System.Diagnostics.Process/README.md b/src/libraries/System.Diagnostics.Process/README.md index 7d221563d97bd6..da3889a238cffb 100644 --- a/src/libraries/System.Diagnostics.Process/README.md +++ b/src/libraries/System.Diagnostics.Process/README.md @@ -12,26 +12,17 @@ See the [Help Wanted](https://github.com/dotnet/runtime/issues?q=is%3Aopen+is%3A ## Deployment `System.Diagnostics.Process` is included in the shared framework. The package does not need to be installed into any project compatible with .NET Standard 2.0. -## Temporary hang diagnostics - -This investigation branch enables xUnit's verbose start/finish reporting on Windows, including -theory arguments, without changing test selection or parallelism. These messages are independent -of `XUNIT_HIDE_PASSING_OUTPUT_DIAGNOSTICS`. - -For Windows x86 CoreCLR Helix runs, an eight-minute background watchdog in the xUnit test host -launches that runtime's `createdump.exe --full`. On Windows, this tool dumps its parent process; -it is not launched through a shell and no PID lookup or WER registry configuration is needed. -RemoteExecutor children do not arm the watchdog, and normal runner exit ends the background thread. -An in-flight dump child is terminated and reaped if the runner exits during capture. -Local runs without `HELIX_WORKITEM_UPLOAD_ROOT` do not arm it. - -The dump is written directly to `HELIX_WORKITEM_UPLOAD_ROOT` for Helix artifact collection. -Dump capture has a two-minute budget, followed by at most 30 seconds to terminate the dumper. -This is a snapshot only: successful capture and diagnostic failures do not terminate the test host -or change its exit status. Capture and cleanup failures are logged explicitly to standard error. -The full suite continues until ordinary completion or Helix's original 15-minute deadline, -allowing slow progress to be distinguished from a whole-work-item timeout. - -These diagnostics investigate the unexplained Process-suite timeout independently of earlier -investigations. They are temporary, not a production fix, and do not establish that different -timeout reports share a root cause. +## Helix test partitions + +Windows x86 CoreCLR console-runner test archives are divided into six Helix work items. +Each archive contains the same test assembly and supporting files, with a class filter in its +generated runner script. Each work item has its own payload, results, temporary files, and +test process. The existing assembly-level collection behavior is unchanged within each process. + +The partitions are balanced using matched x86 CI timings, with headroom for slower executions +and work-item setup. The goal is five minutes or less per work item; the normal Helix timeout +is not reduced or increased. All rows of a theory stay together with their class. +The final partition excludes the classes in the first five, so new classes remain covered. + +The ordinary local runner still executes the full suite. Other architectures, Mono, mobile, +NativeAOT, and other single-file runners keep their existing unpartitioned archives. diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessTestHangDiagnostics.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessTestHangDiagnostics.cs deleted file mode 100644 index 0e6e131efd7b02..00000000000000 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessTestHangDiagnostics.cs +++ /dev/null @@ -1,159 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.ComponentModel; -using System.IO; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Text; -using System.Threading; - -namespace System.Diagnostics.Tests; - -internal static class ProcessTestHangDiagnostics -{ - private static readonly TimeSpan s_watchdogTimeout = TimeSpan.FromMinutes(8); - private static readonly TimeSpan s_dumpTimeout = TimeSpan.FromMinutes(2); - private static readonly TimeSpan s_dumperExitTimeout = TimeSpan.FromSeconds(30); - - [ModuleInitializer] - internal static void Initialize() - { - string? uploadRoot = Environment.GetEnvironmentVariable("HELIX_WORKITEM_UPLOAD_ROOT"); - if (string.IsNullOrEmpty(uploadRoot) || - !string.Equals(Path.GetFileName(Environment.GetCommandLineArgs()[0]), "xunit.console.dll", StringComparison.OrdinalIgnoreCase)) - { - return; - } - - string dumpTool = Path.Combine(RuntimeEnvironment.GetRuntimeDirectory(), "createdump.exe"); - var watchdog = new Thread(() => Watchdog(dumpTool, uploadRoot, s_watchdogTimeout, s_dumpTimeout)) - { - // The watchdog belongs to the runner, not to RemoteExecutor children, and ends with the runner. - IsBackground = true, - Name = "Process tests hang watchdog" - }; - watchdog.Start(); - } - - private static void Watchdog(string dumpTool, string uploadRoot, TimeSpan watchdogTimeout, TimeSpan dumpTimeout) - { - // Bypass xUnit's per-test output capture, including passing-output suppression. - using TextWriter log = TextWriter.Synchronized(new StreamWriter(Console.OpenStandardError(), new UTF8Encoding(false), leaveOpen: true) - { - AutoFlush = true - }); - log.WriteLine($"[Process hang diagnostics] Armed for {watchdogTimeout}; PID={Environment.ProcessId}; architecture={RuntimeInformation.ProcessArchitecture}; OS={Environment.OSVersion}; createdump={dumpTool}; upload={uploadRoot}"); - - Thread.Sleep(watchdogTimeout); - string dumpPath = Path.Combine(uploadRoot, $"ProcessTests.{Environment.ProcessId}.dmp"); - log.WriteLine($"[Process hang diagnostics] Snapshot time reached. Capturing full test-host dump to {dumpPath} without terminating tests."); - try - { - Directory.CreateDirectory(uploadRoot); - CaptureDump(dumpTool, dumpPath, dumpTimeout, log); - } - catch (Exception e) when (e is IOException or UnauthorizedAccessException or Win32Exception or InvalidOperationException or OperationCanceledException) - { - log.WriteLine($"[Process hang diagnostics] Dump capture failed: {e}"); - } - } - - private static void CaptureDump(string dumpTool, string dumpPath, TimeSpan dumpTimeout, TextWriter log) - { - var startInfo = new ProcessStartInfo(dumpTool) - { - UseShellExecute = false - }; - startInfo.ArgumentList.Add("--full"); - startInfo.ArgumentList.Add("--name"); - startInfo.ArgumentList.Add(dumpPath); - - using var dumper = new Process { StartInfo = startInfo }; - var exitLock = new object(); - bool started = false; - bool finished = false; - bool hostExiting = false; - - AppDomain.CurrentDomain.ProcessExit += OnProcessExit; - try - { - lock (exitLock) - { - if (hostExiting) - { - throw new OperationCanceledException("The test host exited before the snapshot could start."); - } - - // Windows createdump targets its parent, so start it directly in the test host. - started = dumper.Start(); - if (!started) - { - throw new InvalidOperationException("Could not start createdump."); - } - } - - log.WriteLine($"[Process hang diagnostics] createdump PID={dumper.Id} started."); - if (!dumper.WaitForExit((int)dumpTimeout.TotalMilliseconds)) - { - throw new IOException($"createdump exceeded its {dumpTimeout} budget."); - } - - if (dumper.ExitCode != 0 || !File.Exists(dumpPath) || new FileInfo(dumpPath).Length == 0) - { - throw new IOException($"createdump exited with code {dumper.ExitCode} without a successful nonempty dump at {dumpPath}."); - } - - log.WriteLine($"[Process hang diagnostics] Full dump captured: {dumpPath} ({new FileInfo(dumpPath).Length} bytes). Test execution continues."); - } - finally - { - lock (exitLock) - { - finished = true; - AppDomain.CurrentDomain.ProcessExit -= OnProcessExit; - if (started) - { - StopDumper(dumper); - } - } - } - - void OnProcessExit(object? sender, EventArgs args) - { - lock (exitLock) - { - if (finished) - { - return; - } - - hostExiting = true; - if (started) - { - try - { - StopDumper(dumper); - } - catch (Exception e) when (e is Win32Exception or InvalidOperationException) - { - log.WriteLine($"[Process hang diagnostics] Dump child cleanup failed during test-host exit: {e}"); - } - } - } - } - } - - private static void StopDumper(Process dumper) - { - if (!dumper.HasExited) - { - dumper.Kill(); - } - - if (!dumper.WaitForExit((int)s_dumperExitTimeout.TotalMilliseconds)) - { - throw new InvalidOperationException($"createdump PID {dumper.Id} did not exit after termination."); - } - } -} diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessTestShards.targets b/src/libraries/System.Diagnostics.Process/tests/ProcessTestShards.targets new file mode 100644 index 00000000000000..4559cb886cfc63 --- /dev/null +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessTestShards.targets @@ -0,0 +1,73 @@ + + + <_ProcessTestShardDefinition>$(MSBuildThisFileFullPath) + <_ShardProcessTests Condition="'$(ArchiveTests)' == 'true' and + '$(TargetPlatformIdentifier)' == 'windows' and + '$(TargetArchitecture)' == 'x86' and + '$(RuntimeFlavor)' == 'CoreCLR' and + '$(TestSingleFile)' != 'true'">true + + + + <_ProcessTestShard Include="1" Arguments="-class System.Diagnostics.Tests.ProcessMultiplexingTests" /> + <_ProcessTestShard Include="2" Arguments="-class System.Diagnostics.Tests.ProcessStreamingTests -class System.Diagnostics.Tests.SafeProcessHandleTests" /> + <_ProcessTestShard Include="3" Arguments="-class System.Diagnostics.Tests.ProcessTests -class System.Diagnostics.Tests.StartSuspendedTests -class System.Diagnostics.Tests.StartAndForgetTests -class System.Diagnostics.Tests.ProcessCollectionTests -class System.Diagnostics.Tests.ProcessExitStatusTests" /> + <_ProcessTestShard Include="4" Arguments="-class System.Diagnostics.Tests.ProcessHandlesTests -class System.Diagnostics.Tests.ProcessStartInfoTests" /> + <_ProcessTestShard Include="5" Arguments="-class System.Diagnostics.Tests.RunTests -class System.Diagnostics.Tests.ProcessWaitingTests -class System.Diagnostics.Tests.ProcessOpenTests -class System.Diagnostics.Tests.ProcessStandardConsoleTests -class System.Diagnostics.Tests.ProcessModuleTests -class System.Diagnostics.Tests.ProcessThreadTests" /> + + <_ProcessTestShard Include="6" Arguments="-noclass System.Diagnostics.Tests.ProcessMultiplexingTests -noclass System.Diagnostics.Tests.ProcessStreamingTests -noclass System.Diagnostics.Tests.SafeProcessHandleTests -noclass System.Diagnostics.Tests.ProcessTests -noclass System.Diagnostics.Tests.StartSuspendedTests -noclass System.Diagnostics.Tests.StartAndForgetTests -noclass System.Diagnostics.Tests.ProcessCollectionTests -noclass System.Diagnostics.Tests.ProcessExitStatusTests -noclass System.Diagnostics.Tests.ProcessHandlesTests -noclass System.Diagnostics.Tests.ProcessStartInfoTests -noclass System.Diagnostics.Tests.RunTests -noclass System.Diagnostics.Tests.ProcessWaitingTests -noclass System.Diagnostics.Tests.ProcessOpenTests -noclass System.Diagnostics.Tests.ProcessStandardConsoleTests -noclass System.Diagnostics.Tests.ProcessModuleTests -noclass System.Diagnostics.Tests.ProcessThreadTests" /> + + + + + <_ProcessTestPayload Include="$(OutDir)**\*" Exclude="$(RunScriptOutputPath)" /> + + + + + + <_ProcessTestShardDirectory>$(IntermediateOutputPath)process-test-shards\%(_ProcessTestShard.Identity)\ + + + <_ProcessTestShardRunCommands Remove="@(_ProcessTestShardRunCommands)" /> + <_ProcessTestShardRunCommands Include="$([System.IO.File]::ReadAllText('$(PreExecutionTestScript)'))" + Condition="'$(PreExecutionTestScript)' != ''" /> + <_ProcessTestShardRunCommands Include="$(RunScriptCommand) %(_ProcessTestShard.Arguments)" /> + <_ProcessTestShardRunCommands Include="@(PostRunScriptCommands)" /> + + + + + + + + + + + + + + + + + + diff --git a/src/libraries/System.Diagnostics.Process/tests/System.Diagnostics.Process.Tests.csproj b/src/libraries/System.Diagnostics.Process/tests/System.Diagnostics.Process.Tests.csproj index f83b32ce5d202d..d3ee2f4468f95c 100644 --- a/src/libraries/System.Diagnostics.Process/tests/System.Diagnostics.Process.Tests.csproj +++ b/src/libraries/System.Diagnostics.Process/tests/System.Diagnostics.Process.Tests.csproj @@ -10,11 +10,7 @@ $([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) $(DefineConstants);TargetsWindows - true - - - @@ -123,4 +119,5 @@ +