Derive ilasm -DET identities from the assembly's content

src/coreclr/dlls/mscorpe/ceefilegenwriter.cpp

@@ -406,6 +406,19 @@ HRESULT CeeFileGenWriter::generateImage(void **ppImage)
     return hr;
 } // HRESULT CeeFileGenWriter::generateImage()
 
+// Hash the image as generateImage would write it now. Fixing up first means the hash covers the
+// relocated section contents and the final headers, and generateImage does not fix up again.
+HRESULT CeeFileGenWriter::computeImageHash(HRESULT (*computeHash)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize),
+                                           BYTE* pHash, DWORD hashSize)
+{
+    HRESULT hr;
+
+    if (!m_fixed)
+        IfFailRet(fixup());
+
+    return getPEWriter().computeImageHash(computeHash, pHash, hashSize);
+} // HRESULT CeeFileGenWriter::computeImageHash()
+
 HRESULT CeeFileGenWriter::setOutputFileName(_In_ LPWSTR fileName)
 {
     if (m_outputFileName)

src/coreclr/dlls/mscorpe/iceefilegen.cpp

@@ -457,3 +457,14 @@ HRESULT ICeeFileGen::SetFileHeaderTimeStamp(HCEEFILE ceeFile, DWORD timeStamp)
     return S_OK;
 }
 
+HRESULT ICeeFileGen::ComputeImageHash(HCEEFILE ceeFile,
+                                      HRESULT (*computeHash)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize),
+                                      BYTE* pHash, DWORD hashSize)
+{
+    TESTANDRETURNPOINTER(ceeFile);
+    TESTANDRETURNPOINTER(computeHash);
+    TESTANDRETURNPOINTER(pHash);
+
+    CeeFileGenWriter *gen = reinterpret_cast<CeeFileGenWriter*>(ceeFile);
+    return gen->computeImageHash(computeHash, pHash, hashSize);
+}

src/coreclr/dlls/mscorpe/pewriter.cpp

@@ -1445,13 +1445,25 @@ HRESULT PEWriter::write(_In_ LPCWSTR fileName) {
     return hr;
 }
 
-HRESULT PEWriter::write(void ** ppImage)
+// The size of the file that write(fileName) produces. That is filePos, unless a section's data
+// extends beyond it: with stripped relocations, the .reloc section is still written, but its
+// size is not counted in filePos.
+size_t PEWriter::getImageSize()
 {
-    const unsigned RoundUpVal = VAL32(m_ntHeaders->OptionalHeader.FileAlignment);
-    char *pad = (char *) _alloca(RoundUpVal);
-    memset(pad, 0, RoundUpVal);
+    size_t size = filePos;
+    for (PEWriterSection **cur = getSectStart(); cur < getSectCur(); cur++) {
+        if ((*cur)->m_header != NULL) {
+            size_t sectionEnd = (size_t)(*cur)->m_filePos + (*cur)->dataLen() + (*cur)->m_filePad;
+            if (sectionEnd > size)
+                size = sectionEnd;
+        }
+    }
+    return size;
+}
 
-    size_t lSize = filePos;
+HRESULT PEWriter::write(void ** ppImage)
+{
+    size_t lSize = getImageSize();
 
     // allocate the block we are handing back to the caller
     void * pImage = (void *) CoTaskMemAlloc(lSize);
@@ -1480,7 +1492,7 @@ HRESULT PEWriter::write(void ** ppImage)
             pCur = (char*)pImage + (*cur)->m_filePos;
             len = (*cur)->writeMem((void**)&pCur);
             _ASSERTE(len == (*cur)->dataLen());
-            COPY_AND_ADVANCE(pCur, pad, (*cur)->m_filePad);
+            pCur += (*cur)->m_filePad; // the image is already zeroed
         }
     }
 
@@ -1494,6 +1506,24 @@ HRESULT PEWriter::write(void ** ppImage)
     return S_OK;
 }
 
+// Hash the image that write() would produce now. Must be called after fixup().
+HRESULT PEWriter::computeImageHash(HRESULT (*computeHash)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize),
+                                   BYTE* pHash, DWORD hashSize)
+{
+    size_t imageSize = getImageSize();
+    if (imageSize > UINT32_MAX)
+        return COR_E_OVERFLOW;
+
+    void *pImage = NULL;
+    HRESULT hr = write(&pImage);
+    if (FAILED(hr))
+        return hr;
+
+    hr = computeHash((BYTE*)pImage, static_cast<DWORD>(imageSize), pHash, hashSize);
+    CoTaskMemFree(pImage);
+    return hr;
+}
+
 HRESULT PEWriter::getFileTimeStamp(DWORD *pTimeStamp)
 {
     if (pTimeStamp)

src/coreclr/dlls/mscorpe/pewriter.h

@@ -40,6 +40,8 @@ class PEWriter : public PESectionMan
     HRESULT fixup(CeeGenTokenMapper *pMapper);
     HRESULT write(_In_ LPCWSTR fileName);
     HRESULT write(void **ppImage);
+    HRESULT computeImageHash(HRESULT (*computeHash)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize),
+                             BYTE* pHash, DWORD hashSize);
 
     // calling these functions is optional
     DWORD      getSectionAlignment();
@@ -125,6 +127,8 @@ class PEWriter : public PESectionMan
         return COUNT_T(sectCur - sectStart);
     }
 
+    size_t getImageSize();
+
 
     IMAGE_DOS_HEADER    m_dosHeader;
     IMAGE_NT_HEADERS *  m_ntHeaders;

src/coreclr/ilasm/portable_pdb.cpp

@@ -160,15 +160,36 @@ HRESULT PortablePdbWriter::BuildPdbStream(IMetaDataEmit3* peEmitter, mdMethodDef
     return hr;
 }
 
-HRESULT PortablePdbWriter::ComputeSha256PdbStreamChecksum(BYTE(&checksum)[32])
+HRESULT PortablePdbWriter::ComputeSha256PdbChecksum(BYTE(&checksum)[32])
 {
-    return m_ilasmPdbWriter->ComputeSha256PdbStreamChecksum(Sha256Hash, checksum);
+    // The checksum is the hash of the entire PDB file with its 20-byte PDB ID zeroed
+    // (see "PDB Checksum Debug Directory Entry" in docs/design/specs/PE-COFF.md).
+    // Serialize the PDB the same way it is saved later, with the ID temporarily zeroed,
+    // then restore the ID.
+    HRESULT hr = S_OK;
+    HRESULT restoreHr = S_OK;
+    DWORD pdbSize = 0;
+    BYTE* pdbData = NULL;
+
+    if (FAILED(hr = m_ilasmPdbWriter->ChangePdbStreamId(GUID(), 0))) goto exit;
+    if (FAILED(hr = m_pdbEmitter->GetSaveSize(cssAccurate, &pdbSize))) goto exit;
+    pdbData = new BYTE[pdbSize];
+    if (FAILED(hr = m_pdbEmitter->SaveToMemory(pdbData, pdbSize))) goto exit;
+    hr = Sha256Hash(pdbData, pdbSize, checksum, sizeof(checksum));
+
+exit:
+    restoreHr = m_ilasmPdbWriter->ChangePdbStreamId(m_pdbStream.id.pdbGuid, m_pdbStream.id.pdbTimeStamp);
+    if (SUCCEEDED(hr))
+        hr = restoreHr;
+    delete[] pdbData;
+    return hr;
 }
 
-HRESULT PortablePdbWriter::ChangePdbStreamGuid(REFGUID newGuid)
+HRESULT PortablePdbWriter::ChangePdbStreamId(REFGUID newGuid, const ULONG newTimestamp)
 {
     m_pdbStream.id.pdbGuid = newGuid;
-    return m_ilasmPdbWriter->ChangePdbStreamGuid(newGuid);
+    m_pdbStream.id.pdbTimeStamp = newTimestamp;
+    return m_ilasmPdbWriter->ChangePdbStreamId(newGuid, newTimestamp);
 }
 
 HRESULT PortablePdbWriter::DefineDocument(char* name, GUID* language)

src/coreclr/ilasm/portable_pdb.h

@@ -50,8 +50,8 @@ class PortablePdbWriter
     void            SetTimestamp(const ULONG newTimestamp);
     Document*       GetCurrentDocument();
     HRESULT         BuildPdbStream(IMetaDataEmit3* peEmitter, mdMethodDef entryPoint);
-    HRESULT         ComputeSha256PdbStreamChecksum(BYTE (&checksum)[32]);
-    HRESULT         ChangePdbStreamGuid(REFGUID newGuid);
+    HRESULT         ComputeSha256PdbChecksum(BYTE (&checksum)[32]);
+    HRESULT         ChangePdbStreamId(REFGUID newGuid, const ULONG newTimestamp);
     HRESULT         DefineDocument(char* name, GUID* language);
     HRESULT         DefineSequencePoints(Method* method);
     HRESULT         DefineLocalScope(Method* method);

src/coreclr/ilasm/writer.cpp

@@ -79,8 +79,8 @@ HRESULT Assembler::InitMetaData()
 
         if (m_fDeterministic)
         {
-            // When build determinism is enabled, the PDB checksum is computed with these fields set to zero.
-            // The GUID and timestamp will be updated after.
+            // In deterministic mode, the random GUID and timestamp set by Init() must not reach the output.
+            // The deterministic PDB ID is written in CreatePEFile, once the PDB content hash is known.
             m_pPortablePdbWriter->SetGuid(GUID());
             m_pPortablePdbWriter->SetTimestamp(0);
         }
@@ -578,9 +578,9 @@ HRESULT Assembler::CreateExportDirectory()
     // Data blob is ready pending Name Pointer Table values offsetting
 
     memset(&exportDirIDD,0,sizeof(IMAGE_EXPORT_DIRECTORY));
-    // Grab the timestamp of the PE file.
-    DWORD fileTimeStamp;
-    if (FAILED(hr = m_pCeeFileGen->GetFileTimeStamp(m_pCeeFile,&fileTimeStamp))) return hr;
+    // Grab the timestamp of the PE file. It is the current time, so deterministic output uses 0.
+    DWORD fileTimeStamp = 0;
+    if (!m_fDeterministic && FAILED(hr = m_pCeeFileGen->GetFileTimeStamp(m_pCeeFile,&fileTimeStamp))) return hr;
     // Fill in the directory entry.
     // Characteristics, MajorVersion and MinorVersion play no role and stay 0
     exportDirIDD.TimeDateStamp = VAL32(fileTimeStamp);
@@ -1162,14 +1162,50 @@ HRESULT Assembler::AllocateStrongNameSignature()
     return S_OK;
 }
 
+// Write the MVID into metadata that has already been serialized into the image.
+// The Module row's Mvid column (ECMA-335 II.22.30) is a 1-based index into the #GUID heap (II.24.2.5).
+static HRESULT SetSerializedMvid(IMetaDataEmit3* pEmitter, BYTE* metaData, DWORD metaDataSize, REFGUID mvid)
+{
+    HRESULT hr;
+    IMetaDataTables* pTables = NULL;
+    ULONG guidIndex = 0;
+
+    if (FAILED(hr = pEmitter->QueryInterface(IID_IMetaDataTables, (void**)&pTables))) return hr;
+    hr = pTables->GetColumn(TBL_Module, ModuleRec::COL_Mvid, 1, &guidIndex);
+    pTables->Release();
+    if (FAILED(hr)) return hr;
+
+    STORAGEHEADER header;
+    ULONG cbStreamHeaders = metaDataSize;
+    PSTORAGESTREAM pStream = MDFormat::GetFirstStream_Verify(&header, metaData, &cbStreamHeaders);
+    if (pStream == NULL) return E_FAIL;
+    BYTE* pEnd = metaData + metaDataSize;
+    for (USHORT i = 0; i < header.GetiStreams(); i++, pStream = pStream->NextStream())
+    {
+        BYTE* pName = (BYTE*)pStream + 2 * sizeof(ULONG);
+        if ((pName >= pEnd) || (memchr(pName, 0, pEnd - pName) == NULL))
+            return E_FAIL;
+
+        if (strcmp(pStream->GetName(), "#GUID") == 0)
+        {
+            if ((pStream->GetOffset() > metaDataSize) || (pStream->GetSize() > metaDataSize - pStream->GetOffset()) ||
+                (guidIndex == 0) || (guidIndex > pStream->GetSize() / sizeof(GUID)))
+                return E_FAIL;
+
+            BYTE* pMvid = metaData + pStream->GetOffset() + (guidIndex - 1) * sizeof(GUID);
+            memcpy_s(pMvid, sizeof(GUID), &mvid, sizeof(GUID));
+            return S_OK;
+        }
+    }
+    return E_FAIL;
+}
+
 HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename)
 {
     HRESULT             hr;
     DWORD               mresourceSize = 0;
     BYTE*               mresourceData = NULL;
     WCHAR*              wzScopeName = NULL;
-    GUID                deterministicGuid = GUID();
-    ULONG               deterministicTimestamp = 0;
 
     if(bClock) bClock->cMDEmitBegin = minipal_lowres_ticks();
     if(m_fReportProgress) printf("Creating PE file\n");
@@ -1455,21 +1491,13 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename)
 
     if (m_fDeterministic)
     {
-        // Get deterministic GUID and timestamp from the computed hash
-        BYTE hash[32];
-        _ASSERTE(sizeof(GUID) + sizeof(ULONG) <= sizeof(hash));
-        if (FAILED(hr = Sha256Hash(metaData, metaDataSize, hash, sizeof(hash)))) goto exit;
-
-        memcpy_s(&deterministicGuid, sizeof(GUID), hash, sizeof(GUID));
-        memcpy_s(&deterministicTimestamp, sizeof(ULONG), hash + sizeof(GUID), sizeof(ULONG));
-
-        // In deterministic mode, the MVID needs to be stabilized for the metadata scope that was
-        // created in Assembler::InitMetaData, and it is guaranteed that the IMDInternalEmit for
-        // that scope was already acquired immediately after that scope was created.
+        // In deterministic mode, the MVID and the PE timestamp are derived from the content of the
+        // image once it is complete, at the end of this method. Until then both are zero, so that
+        // neither the random MVID set when the scope was created nor the current time is hashed.
+        // The IMDInternalEmit for the scope was acquired in Assembler::InitMetaData.
         _ASSERTE(m_pInternalEmitForDeterministicMvid != NULL);
-        m_pInternalEmitForDeterministicMvid->ChangeMvid(deterministicGuid);
-
-        if (FAILED(hr = m_pCeeFileGen->SetFileHeaderTimeStamp(m_pCeeFile, deterministicTimestamp))) goto exit;
+        if (FAILED(hr = m_pInternalEmitForDeterministicMvid->ChangeMvid(GUID()))) goto exit;
+        if (FAILED(hr = m_pCeeFileGen->SetFileHeaderTimeStamp(m_pCeeFile, 0))) goto exit;
     }
 
     if (m_fGeneratePDB)
@@ -1480,16 +1508,21 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename)
         if (FAILED(hr = m_pPortablePdbWriter->BuildPdbStream(m_pEmitter, entryPoint))) goto exit;
 
         BYTE pdbChecksum[32];
-        if (FAILED(hr = m_pPortablePdbWriter->ComputeSha256PdbStreamChecksum(pdbChecksum))) goto exit;
+        if (FAILED(hr = m_pPortablePdbWriter->ComputeSha256PdbChecksum(pdbChecksum))) goto exit;
 
         if (m_fDeterministic)
         {
-            // Now that the PDB checksum has been computed, update the GUID and timestamp
+            // The checksum is the hash of the PDB with its ID zeroed, and the PDB is otherwise final,
+            // so the deterministic PDB ID is derived from it (see docs/design/specs/PE-COFF.md).
+            // The CodeView entry created below carries this ID, so the PE hash covers it.
             _ASSERTE(*(m_pPortablePdbWriter->GetGuid()) == GUID());
-            if (FAILED(hr = m_pPortablePdbWriter->ChangePdbStreamGuid(deterministicGuid))) goto exit;
-
             _ASSERTE(m_pPortablePdbWriter->GetTimestamp() == 0);
-            m_pPortablePdbWriter->SetTimestamp(deterministicTimestamp);
+            GUID pdbGuid;
+            ULONG pdbTimestamp;
+            _ASSERTE(sizeof(GUID) + sizeof(ULONG) <= sizeof(pdbChecksum));
+            memcpy_s(&pdbGuid, sizeof(GUID), pdbChecksum, sizeof(GUID));
+            memcpy_s(&pdbTimestamp, sizeof(ULONG), pdbChecksum + sizeof(GUID), sizeof(ULONG));
+            if (FAILED(hr = m_pPortablePdbWriter->ChangePdbStreamId(pdbGuid, pdbTimestamp))) goto exit;
         }
 
         if (FAILED(hr=CreateDebugDirectory(pdbChecksum))) goto exit;
@@ -1788,6 +1821,27 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename)
         }
     }
 
+    if (m_fDeterministic)
+    {
+        // The image is now complete apart from its MVID and PE timestamp, which are zero.
+        // Derive both from the hash of the image as it will be written: the headers, the IL,
+        // the metadata, the resources and the debug directory, which carries the PDB ID.
+        // The image is fixed up before it is hashed, so GenerateCeeFile writes these same bytes,
+        // apart from the MVID and the timestamp set here.
+        BYTE hash[32];
+        _ASSERTE(sizeof(GUID) + sizeof(ULONG) <= sizeof(hash));
+        if (FAILED(hr = m_pCeeFileGen->ComputeImageHash(m_pCeeFile, Sha256Hash, hash, sizeof(hash)))) goto exit;
+
+        GUID deterministicGuid;
+        ULONG deterministicTimestamp;
+        memcpy_s(&deterministicGuid, sizeof(GUID), hash, sizeof(GUID));
+        memcpy_s(&deterministicTimestamp, sizeof(ULONG), hash + sizeof(GUID), sizeof(ULONG));
+
+        // The metadata has already been serialized into the image, so the MVID is written there.
+        if (FAILED(hr = SetSerializedMvid(m_pEmitter, metaData, metaDataSize, deterministicGuid))) goto exit;
+        if (FAILED(hr = m_pCeeFileGen->SetFileHeaderTimeStamp(m_pCeeFile, deterministicTimestamp))) goto exit;
+    }
+
     hr = S_OK;
 
 exit:

src/coreclr/inc/ceefilegenwriter.h

@@ -79,6 +79,8 @@ class CeeFileGenWriter : public CCeeGen
     HRESULT link();     // Layout the sections and assign their starting addresses
     HRESULT fixup();    // Apply relocations to any pointer data. Also generate PE base relocs
     HRESULT generateImage(void **ppImage);
+    HRESULT computeImageHash(HRESULT (*computeHash)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize),
+                             BYTE* pHash, DWORD hashSize);
 
     HRESULT setImageBase(size_t imageBase);
     HRESULT setImageBase64(ULONGLONG imageBase);

src/coreclr/inc/iceefilegen.h

@@ -170,6 +170,13 @@ class ICeeFileGen {
                                                      ULONG *pNumSections);
 
     virtual HRESULT SetVTableEntry64(HCEEFILE ceeFile, ULONG size, void* ptr);
+
+    // Fix up the image and hash it as GenerateCeeFile would write it now. The caller may then
+    // change bytes it owns (such as the MVID) and the file header timestamp before GenerateCeeFile,
+    // but must not add section data or relocations, which would no longer be applied.
+    virtual HRESULT ComputeImageHash(HCEEFILE ceeFile,
+                                     HRESULT (*computeHash)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize),
+                                     BYTE* pHash, DWORD hashSize);
 };
 
 #endif

src/coreclr/md/compiler/emit.cpp

@@ -2116,29 +2116,16 @@ STDMETHODIMP RegMeta::DefineLocalVariable(      // S_OK or error.
 } // RegMeta::DefineLocalVariable
 
 //*******************************************************************************
-// ComputeSha256PdbStreamChecksum
+// ChangePdbStreamId
 //*******************************************************************************
-STDMETHODIMP RegMeta::ComputeSha256PdbStreamChecksum(
-        HRESULT (*computeSha256)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize),
-        BYTE (&checksum)[32])
+STDMETHODIMP RegMeta::ChangePdbStreamId(
+        REFGUID newGuid,
+        ULONG newTimestamp)
 {
 #ifdef FEATURE_METADATA_EMIT_IN_DEBUGGER
     return E_NOTIMPL;
 #else //!FEATURE_METADATA_EMIT_IN_DEBUGGER
-    return m_pStgdb->m_pPdbHeap->ComputeSha256Checksum(computeSha256, checksum);
-#endif //!FEATURE_METADATA_EMIT_IN_DEBUGGER
-}
-
-//*******************************************************************************
-// ChangePdbStreamGuid
-//*******************************************************************************
-STDMETHODIMP RegMeta::ChangePdbStreamGuid(
-        REFGUID newGuid)
-{
-#ifdef FEATURE_METADATA_EMIT_IN_DEBUGGER
-    return E_NOTIMPL;
-#else //!FEATURE_METADATA_EMIT_IN_DEBUGGER
-    return m_pStgdb->m_pPdbHeap->SetDataGuid(newGuid);
+    return m_pStgdb->m_pPdbHeap->SetDataId(newGuid, newTimestamp);
 #endif //!FEATURE_METADATA_EMIT_IN_DEBUGGER
 }