Write the managed ilasm PDB to a separate file, as native ilasm does

src/tests/Common/CLRTest.Jit.targets

@@ -430,9 +430,11 @@ for inputAssemblyName in glob.glob("*.dll"):
       if not is_managed_debug_assembly(inputAssemblyName):
         ilasmSwitches = ilasmSwitches + " -DEBUG"
 
-      pdbName = inputAssemblyName.replace('.dll', '.pdb')
+      # Assemble to an image of its own and hash that image and the PDB that ilasm writes beside it.
+      pdbCheckAssemblyName = os.path.join("IL-RT", assemblyNameWithoutExtension + ".pdbcheck.dll")
+      pdbName = os.path.join("IL-RT", assemblyNameWithoutExtension + ".pdbcheck.pdb")
 
-      ilasm_args = f'{ilasm_path} -output={pdbName} {ilasmSwitches} {disassemblyName}'
+      ilasm_args = f'{ilasm_path} -output={pdbCheckAssemblyName} {ilasmSwitches} {disassemblyName}'
       print(ilasm_args)
       proc = subprocess.Popen(ilasm_args, shell=True)
 
@@ -446,9 +448,10 @@ for inputAssemblyName in glob.glob("*.dll"):
           print(f"ILASM failed with exit code {proc.returncode}")
           sys.exit(1)
 
+      imageHash = hash_file(pdbCheckAssemblyName)
       hash = hash_file(pdbName)
 
-      ilasm_args = f'{ilasm_path} -output={pdbName} {ilasmSwitches} {disassemblyName}'
+      ilasm_args = f'{ilasm_path} -output={pdbCheckAssemblyName} {ilasmSwitches} {disassemblyName}'
       print(ilasm_args)
       proc = subprocess.Popen(ilasm_args, shell=True)
 
@@ -462,6 +465,10 @@ for inputAssemblyName in glob.glob("*.dll"):
           print(f"ILASM failed with exit code {proc.returncode}")
           sys.exit(1)
 
+      if imageHash != hash_file(pdbCheckAssemblyName):
+        print("ILASM determinism with -DEBUG failed")
+        sys.exit(1)
+
       if hash != hash_file(pdbName):
         print("ILASM PDB determinism failed")
         sys.exit(1)

src/tools/ilasm/src/ILAssembler/Actions/GrammarActions.BuildImage.cs

@@ -137,8 +137,8 @@ internal sealed partial class GrammarActions
                 entryPoint = (MethodDefinitionHandle)_entityRegistry.EntryPoint.Handle;
             }
 
-            // Build debug directory if we have any debug info
-            DebugDirectoryBuilder? debugDirectoryBuilder = BuildDebugDirectory(entryPoint, out _);
+            // Build the PDB and the debug directory that references it, if a PDB is requested
+            DebugDirectoryBuilder? debugDirectoryBuilder = BuildDebugDirectory(entryPoint, out ImmutableArray<byte>? portablePdb);
 
             Func<IEnumerable<Blob>, BlobContentId>? deterministicIdProvider = _options.Deterministic
                 ? GetDeterministicContentId
@@ -177,7 +177,7 @@ internal sealed partial class GrammarActions
                     exports: exports,
                     dataLabelFixups: validatedDataLabelFixups);
 
-                return (_diagnostics.ToImmutable(), new CompilationResult(peBuilder, mvidFixup));
+                return (_diagnostics.ToImmutable(), new CompilationResult(peBuilder, mvidFixup, portablePdb));
             }
 
             // Apply CorFlags from options or directive
@@ -187,7 +187,7 @@ internal sealed partial class GrammarActions
                 standardCorFlags |= CorFlags.Prefers32Bit;
             }
 
-            ManagedPEBuilder standardBuilder = new(
+            ILAssemblerPEBuilder standardBuilder = new(
                 header,
                 rootBuilder,
                 ilStream,
@@ -198,18 +198,21 @@ internal sealed partial class GrammarActions
                 debugDirectoryBuilder: debugDirectoryBuilder,
                 deterministicIdProvider: deterministicIdProvider);
 
-            return (_diagnostics.ToImmutable(), new CompilationResult(standardBuilder, mvidFixup));
+            return (_diagnostics.ToImmutable(), new CompilationResult(standardBuilder, mvidFixup, portablePdb));
         }
 
         private static BlobContentId GetDeterministicContentId(IEnumerable<Blob> content)
+            => BlobContentId.FromHash(ComputeSha256(content));
+
+        private static byte[] ComputeSha256(IEnumerable<Blob> content)
         {
             using IncrementalHash hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
             foreach (Blob blob in content)
             {
                 hash.AppendData(blob.GetBytes());
             }
 
-            return BlobContentId.FromHash(hash.GetHashAndReset());
+            return hash.GetHashAndReset();
         }
 
         private ImmutableArray<ValidatedVTableFixup> ValidateVTableFixups(Machine machine)
@@ -554,24 +557,41 @@ private ImmutableArray<ValidatedExport> ValidateExports(
             }
         }
 
-        private DebugDirectoryBuilder? BuildDebugDirectory(MethodDefinitionHandle entryPoint, out int debugDataSize)
+        /// <summary>
+        /// Builds the Portable PDB and the debug directory that references it, when a PDB is requested.
+        /// </summary>
+        /// <remarks>
+        /// <para>
+        /// A PDB is produced when <see cref="Options.Debug"/>, <see cref="Options.DebugMode"/> or
+        /// <see cref="Options.Pdb"/> is set; <c>.line</c> directives alone do not produce one. Otherwise this
+        /// returns <see langword="null"/>, the image has no debug directory (<see cref="ILAssemblerPEBuilder"/>),
+        /// and <paramref name="portablePdb"/> is <see langword="null"/>.
+        /// </para>
+        /// <para>
+        /// The PDB is returned in <paramref name="portablePdb"/> for the caller to write as a separate file;
+        /// it is not embedded in the image. The debug directory has these entries, in this order:
+        /// a CodeView entry that names the PDB file (<see cref="GetPdbFilePath"/>) and carries the PDB id;
+        /// a PdbChecksum entry holding the SHA-256 hash of the PDB with its 20-byte id zeroed
+        /// (the PDB Checksum Debug Directory Entry in docs/design/specs/PE-COFF.md); and, with
+        /// <see cref="Options.Deterministic"/>, a Reproducible entry.
+        /// </para>
+        /// <para>
+        /// With <see cref="Options.Deterministic"/>, the PDB id is derived from that same hash
+        /// (<see cref="BlobContentId.FromHash(ImmutableArray{byte})"/>), so the same input gives the same PDB
+        /// bytes. The PDB bytes do not depend on <see cref="Options.PdbFilePath"/>; the image does, because
+        /// the CodeView entry records that path as given, so the same input and the same PDB path give the
+        /// same image bytes, and a PDB path that is a file name alone keeps the image independent of its
+        /// directory. Without <see cref="Options.Deterministic"/>, the id's GUID is random. The checksum is the
+        /// content hash in both cases.
+        /// </para>
+        /// </remarks>
+        private DebugDirectoryBuilder? BuildDebugDirectory(MethodDefinitionHandle entryPoint, out ImmutableArray<byte>? portablePdb)
         {
-            debugDataSize = 0;
+            portablePdb = null;
 
-            // Check if we have any methods with debug info
-            bool hasDebugInfo = false;
-            foreach (var entity in _entityRegistry.GetSeenEntities(TableIndex.MethodDef))
-            {
-                if (entity is EntityRegistry.MethodDefinitionEntity method &&
-                    method.DebugInfo.SequencePoints.Count > 0)
-                {
-                    hasDebugInfo = true;
-                    break;
-                }
-            }
-
-            // Generate PDB if we have debug info OR if --debug/--pdb options are set
-            bool generatePdb = hasDebugInfo || _options.Debug || _options.Pdb;
+            // As in native ilasm, only /DEBUG (any mode) or /PDB produces a PDB. Without them, sequence
+            // points from .line directives are parsed and validated but not emitted.
+            bool generatePdb = _options.Debug || _options.DebugMode is not null || _options.Pdb;
             if (!generatePdb)
             {
                 return null;
@@ -583,40 +603,63 @@ private ImmutableArray<ValidatedExport> ValidateExports(
             // Get row counts from main metadata for the portable PDB
             var typeSystemRowCounts = _metadataBuilder.GetRowCounts();
 
-            Func<IEnumerable<Blob>, BlobContentId> pdbIdProvider = _options.Deterministic
-                ? GetDeterministicContentId
-                : _ => new BlobContentId(Guid.NewGuid(), 0x04030201);
+            // PortablePdbBuilder hands the id provider the serialized PDB with its 20-byte id zeroed.
+            // The SHA-256 of that content is the PdbChecksum value, and the deterministic id.
+            byte[]? pdbContentHash = null;
+            BlobContentId PdbIdProvider(IEnumerable<Blob> content)
+            {
+                pdbContentHash = ComputeSha256(content);
+                return _options.Deterministic
+                    ? BlobContentId.FromHash(pdbContentHash)
+                    : new BlobContentId(Guid.NewGuid(), 0x04030201);
+            }
 
             // Create the portable PDB
             var pdbBuilder = new PortablePdbBuilder(
                 _pdbBuilder,
                 typeSystemRowCounts,
                 entryPoint,
-                idProvider: pdbIdProvider);
+                idProvider: PdbIdProvider);
 
             var pdbBlob = new BlobBuilder();
             var pdbContentId = pdbBuilder.Serialize(pdbBlob);
+            byte[] pdbChecksum = pdbContentHash
+                ?? throw new InvalidOperationException("The Portable PDB was serialized without calling its id provider.");
 
-            // Create debug directory with embedded PDB
+            // Reference the PDB file from the image: CodeView, then PdbChecksum, then (deterministic only)
+            // Reproducible, as native ilasm and the C# compiler do.
             var debugDirectoryBuilder = new DebugDirectoryBuilder();
             debugDirectoryBuilder.AddCodeViewEntry(
-                $"assembly.pdb",
+                GetPdbFilePath(),
                 pdbContentId,
                 pdbBuilder.FormatVersion);
-            debugDirectoryBuilder.AddEmbeddedPortablePdbEntry(pdbBlob, pdbBuilder.FormatVersion);
+            debugDirectoryBuilder.AddPdbChecksumEntry(
+                "SHA256",
+                ImmutableArray.Create(pdbChecksum));
+            if (_options.Deterministic)
+            {
+                debugDirectoryBuilder.AddReproducibleEntry();
+            }
 
-            // Calculate debug data size:
-            // 2 debug directory entries (28 bytes each) + CodeView data (~24 bytes) + Embedded PDB data (compressed pdbBlob + 8 header)
-            // CodeView entry: signature (4) + guid (16) + age (4) + path (variable, ~12 for "assembly.pdb\0")
-            const int debugDirEntrySize = 28;
-            int codeViewDataSize = 4 + 16 + 4 + "assembly.pdb".Length + 1; // signature + guid + age + path + null
-            int embeddedPdbHeaderSize = 8; // MPDB signature (4) + uncompressed size (4)
-            // The embedded PDB is compressed, estimate conservatively as same size
-            int embeddedPdbDataSize = embeddedPdbHeaderSize + pdbBlob.Count;
+            portablePdb = pdbBlob.ToImmutableArray();
+            return debugDirectoryBuilder;
+        }
 
-            debugDataSize = (2 * debugDirEntrySize) + codeViewDataSize + embeddedPdbDataSize;
+        /// <summary>
+        /// Gets the PDB path recorded in the CodeView entry: <see cref="Options.PdbFilePath"/> as given when set,
+        /// otherwise <see cref="Options.OutputFileName"/> with its extension replaced by <c>.pdb</c>,
+        /// otherwise <c>assembly.pdb</c>.
+        /// </summary>
+        private string GetPdbFilePath()
+        {
+            if (_options.PdbFilePath is not null)
+            {
+                return _options.PdbFilePath;
+            }
 
-            return debugDirectoryBuilder;
+            return _options.OutputFileName is not null
+                ? Path.ChangeExtension(_options.OutputFileName, ".pdb")
+                : "assembly.pdb";
         }
 
         private void BuildPdbMetadata()
@@ -744,7 +787,7 @@ private static BlobBuilder EncodeSequencePoints(List<EntityRegistry.SequencePoin
         /// Add DebuggableAttribute to the assembly based on debug options.
         /// - /DEBUG: 0x101 = Default | DisableOptimizations
         /// - /DEBUG=OPT: 0x03 = Default | IgnoreSymbolStoreSequencePoints
-        /// - /DEBUG=IMPL: 0x103 = Default | DisableOptimizations | EnableEditAndContinue
+        /// - /DEBUG=IMPL: 0x103 = Default | IgnoreSymbolStoreSequencePoints | DisableOptimizations
         /// </summary>
         private void ApplyDebuggableAttribute()
         {
@@ -758,7 +801,7 @@ private void ApplyDebuggableAttribute()
             // EnableEditAndContinue = 0x04, DisableOptimizations = 0x100
             const int DebuggingModesDefault = 0x101;  // Default | DisableOptimizations
             const int DebuggingModesOpt = 0x03;       // Default | IgnoreSymbolStoreSequencePoints
-            const int DebuggingModesImpl = 0x103;     // Default | DisableOptimizations | EnableEditAndContinue
+            const int DebuggingModesImpl = 0x103;     // Default | IgnoreSymbolStoreSequencePoints | DisableOptimizations
 
             int debuggingModes = _options.DebugMode switch
             {

src/tools/ilasm/src/ILAssembler/CompilationResult.cs

@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Collections.Immutable;
 using System.Reflection.Metadata;
 using System.Reflection.PortableExecutable;
 
@@ -12,12 +13,28 @@ public sealed class CompilationResult
     private readonly PEBuilder _peBuilder;
     private readonly Blob _mvidFixup;
 
-    internal CompilationResult(PEBuilder peBuilder, Blob mvidFixup)
+    internal CompilationResult(PEBuilder peBuilder, Blob mvidFixup, ImmutableArray<byte>? portablePdb)
     {
         _peBuilder = peBuilder;
         _mvidFixup = mvidFixup;
+        PortablePdb = portablePdb;
     }
 
+    /// <summary>
+    /// Gets the serialized Portable PDB for the image, or <see langword="null"/> when no PDB was requested.
+    /// </summary>
+    /// <remarks>
+    /// A PDB is produced when <see cref="Options.Debug"/>, <see cref="Options.DebugMode"/> or
+    /// <see cref="Options.Pdb"/> is set; <c>.line</c> directives alone do not produce one.
+    /// It is not embedded in the image: the image's debug directory holds a CodeView entry that names the PDB path and carries this PDB's id, followed by a PdbChecksum
+    /// entry with the SHA-256 hash of these bytes with the 20-byte PDB id zeroed, and, with
+    /// <see cref="Options.Deterministic"/>, a Reproducible entry. The PDB path is
+    /// <see cref="Options.PdbFilePath"/> as given when set; otherwise it is <see cref="Options.OutputFileName"/> with its
+    /// extension replaced by <c>.pdb</c>, or <c>assembly.pdb</c> when no output file name is set.
+    /// The caller writes these bytes to the file that path names; a file name alone names a file beside the image.
+    /// </remarks>
+    public ImmutableArray<byte>? PortablePdb { get; }
+
     /// <summary>Serializes the compiled image into the specified builder.</summary>
     /// <param name="builder">The builder that receives the serialized image.</param>
     /// <returns>The content identifier of the serialized image.</returns>

src/tools/ilasm/src/ILAssembler/ILAssemblerPEBuilder.cs

@@ -0,0 +1,65 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Reflection.Metadata;
+using System.Reflection.Metadata.Ecma335;
+using System.Reflection.PortableExecutable;
+
+namespace ILAssembler;
+
+/// <summary>
+/// The <see cref="ManagedPEBuilder"/> for every image the assembler writes. The image has a debug
+/// directory only when the assembler supplies one.
+/// </summary>
+/// <remarks>
+/// Given no <see cref="DebugDirectoryBuilder"/>, a deterministic <see cref="ManagedPEBuilder"/> adds a debug
+/// directory with a Reproducible entry of its own. Native ilasm writes a debug directory only with a PDB, so
+/// this builder passes an empty <see cref="DebugDirectoryBuilder"/> instead, and it clears the PE header's debug
+/// directory entry whenever the directory has no entries, which an empty builder would otherwise leave with a
+/// non-zero address and size 0: an image built without a PDB has no debug directory, deterministic or not.
+/// </remarks>
+internal class ILAssemblerPEBuilder : ManagedPEBuilder
+{
+    public ILAssemblerPEBuilder(
+        PEHeaderBuilder header,
+        MetadataRootBuilder metadataRootBuilder,
+        BlobBuilder ilStream,
+        BlobBuilder? mappedFieldData = null,
+        BlobBuilder? managedResources = null,
+        ResourceSectionBuilder? nativeResources = null,
+        DebugDirectoryBuilder? debugDirectoryBuilder = null,
+        int strongNameSignatureSize = 128,
+        MethodDefinitionHandle entryPoint = default,
+        CorFlags flags = CorFlags.ILOnly,
+        Func<IEnumerable<Blob>, BlobContentId>? deterministicIdProvider = null)
+        : base(
+            header,
+            metadataRootBuilder,
+            ilStream,
+            mappedFieldData,
+            managedResources,
+            nativeResources,
+            debugDirectoryBuilder ?? new DebugDirectoryBuilder(),
+            strongNameSignatureSize,
+            entryPoint,
+            flags,
+            deterministicIdProvider)
+    {
+    }
+
+    protected override PEDirectoriesBuilder GetDirectories()
+    {
+        PEDirectoriesBuilder directories = base.GetDirectories();
+
+        // A debug directory without entries is no debug directory. Clearing it is idempotent, so a later call
+        // (VTableExportPEBuilder calls this again while serializing .reloc) sees the same directories.
+        if (directories.DebugTable.Size == 0)
+        {
+            directories.DebugTable = default;
+        }
+
+        return directories;
+    }
+}