JIT: construct promoted arguments directly

src/coreclr/jit/promotion.cpp

@@ -297,7 +297,11 @@ void AggregateInfoMap::Add(AggregateInfo* agg)
 //
 AggregateInfo* AggregateInfoMap::Lookup(unsigned lclNum)
 {
-    assert(lclNum < m_numLocals);
+    // Temporaries introduced while replacing uses were not promotion candidates.
+    if (lclNum >= m_numLocals)
+    {
+        return nullptr;
+    }
     unsigned index = m_lclNumToAggregateIndex[lclNum];
 
     if (index == UINT_MAX)
@@ -2141,9 +2145,10 @@ void ReplaceVisitor::InsertPreStatementWriteBacks()
                         continue;
                     }
 
-                    if (m_replacer->CanReplaceCallArgWithFieldListOfReplacements(call, &arg, node->AsLclVarCommon()))
+                    if (m_replacer->CanReplaceCallArgWithFieldListOfReplacements(call, &arg, node->AsLclVarCommon()) ||
+                        m_replacer->CanCopyCallArgFromReplacements(call, &arg, node->AsLclVarCommon()))
                     {
-                        // Register arg that can be decomposed into FIELD_LIST.
+                        // The argument can be sourced directly from replacements.
                         continue;
                     }
 
@@ -2399,17 +2404,16 @@ GenTreeFieldList* ReplaceVisitor::CreateFieldListForStructLocal(GenTreeLclVarCom
 
 //------------------------------------------------------------------------
 // ReplaceCallArgWithFieldList:
-//   Handle a call that may pass a struct local with replacements as the
-//   retbuf.
+//   Source a struct argument from its replacements, either as a FIELD_LIST
+//   or by materializing the outgoing copy before global morph.
 //
 // Parameters:
 //   call    - The call
 //   use     - The edge pointing to argNode
 //   argNode - The argument node
 //
 // Returns:
-//   True if the call argument was replaced with a FIELD_LIST; false if the
-//   argument could not be represented as a FIELD_LIST.
+//   True if the argument was replaced; false if write-backs are required.
 //
 bool ReplaceVisitor::ReplaceCallArgWithFieldList(GenTreeCall* call, GenTree** use, GenTreeLclVarCommon* argNode)
 {
@@ -2422,6 +2426,23 @@ bool ReplaceVisitor::ReplaceCallArgWithFieldList(GenTreeCall* call, GenTree** us
 
     if (!CanReplaceCallArgWithFieldListOfReplacements(call, callArg, argNode))
     {
+        if (CanCopyCallArgFromReplacements(call, callArg, argNode))
+        {
+            // Materialize the outgoing copy while the replacement values are
+            // available. Global morph can pass this temporary at its last use
+            // instead of making another copy from synchronized source storage.
+            unsigned temp = m_compiler->lvaGrabTemp(true DEBUGARG("Decomposed struct argument"));
+            m_compiler->lvaSetStruct(temp, argNode->GetLayout(m_compiler), false);
+            GenTree* copy = m_compiler->gtNewStoreLclVarNode(temp, argNode);
+            HandleStructStore(&copy, nullptr);
+            GenTree* value = m_compiler->gtNewLclvNode(temp, TYP_STRUCT);
+            value->gtFlags |= GTF_VAR_DEATH;
+            Statement* copyStmt = m_compiler->fgNewStmtFromTree(copy);
+            m_compiler->fgInsertStmtBefore(m_currentBlock, m_currentStmt, copyStmt);
+            *use          = value;
+            m_madeChanges = true;
+            return true;
+        }
         return false;
     }
 
@@ -2436,6 +2457,48 @@ bool ReplaceVisitor::ReplaceCallArgWithFieldList(GenTreeCall* call, GenTree** us
     return true;
 }
 
+//------------------------------------------------------------------------
+// CanCopyCallArgFromReplacements:
+//   Check whether to materialize an outgoing by-value argument before morph.
+//   Limit this to a standalone direct call with one argument, a dirty non-GC
+//   whole-local source, and a remainder needing at most one primitive copy.
+//   A dying source can already be passed without copying.
+//
+bool ReplaceVisitor::CanCopyCallArgFromReplacements(GenTreeCall* call, CallArg* callArg, GenTreeLclVarCommon* lcl)
+{
+#if FEATURE_IMPLICIT_BYREFS && !defined(UNIX_AMD64_ABI)
+    if (!lcl->OperIs(GT_LCL_VAR) || !callArg->AbiInfo.IsPassedByReference() || (call->gtArgs.CountArgs() != 1) ||
+        (call->gtCallType != CT_USER_FUNC) || (call != m_currentStmt->GetRootNode()) || (callArg->GetNode() != lcl) ||
+        call->IsTailCall() || lcl->GetLayout(m_compiler)->HasGCPtr() ||
+        m_currentBlock->HasPotentialEHSuccs(m_compiler) || IsPromotedStructLocalDying(lcl))
+    {
+        return false;
+    }
+
+    AggregateInfo* agg = m_aggregates.Lookup(lcl->GetLclNum());
+    // Whole-local arguments can reuse the remainder computed during promotion.
+    unsigned remainderSize = agg->UnpromotedMax - agg->UnpromotedMin;
+    bool canCopyRemainder  = (remainderSize == 0) || (isPow2(remainderSize) && (remainderSize <= TARGET_POINTER_SIZE));
+#ifdef FEATURE_SIMD
+    canCopyRemainder |= (remainderSize == 16) && (m_compiler->getPreferredVectorByteLength() >= 16);
+#endif
+    if (canCopyRemainder)
+    {
+        for (const Replacement& rep : agg->Replacements)
+        {
+            if (rep.NeedsWriteBack)
+            {
+                return true;
+            }
+        }
+    }
+    return false;
+#else
+    // The temporary requires fgMarkImplicitByRefCopyOmissionCandidates to avoid another outgoing copy.
+    return false;
+#endif
+}
+
 //------------------------------------------------------------------------
 // CanReplaceCallArgWithFieldListOfReplacements:
 //   Returns true if a struct arg is replaceable by a FIELD_LIST containing

src/coreclr/jit/promotion.h

@@ -298,6 +298,7 @@ class ReplaceVisitor : public GenTreeVisitor<ReplaceVisitor>
     bool ReplaceReturnedStructLocal(GenTreeOp* ret, GenTree** use, GenTreeLclVarCommon* value);
     bool ReplaceCallArgWithFieldList(GenTreeCall* call, GenTree** use, GenTreeLclVarCommon* callArg);
     bool CanReplaceCallArgWithFieldListOfReplacements(GenTreeCall* call, CallArg* callArg, GenTreeLclVarCommon* lcl);
+    bool CanCopyCallArgFromReplacements(GenTreeCall* call, CallArg* callArg, GenTreeLclVarCommon* lcl);
     GenTreeFieldList* CreateFieldListForStructLocal(GenTreeLclVarCommon* value);
     void              ReadBackAfterCall(GenTreeCall* call, GenTree* user);
     bool              IsPromotedStructLocalDying(GenTreeLclVarCommon* structLcl);

src/tests/JIT/Directed/Directed_do.csproj

@@ -10,6 +10,7 @@
     <MergedWrapperProjectReference Include="Convert\out_of_range_fp_to_int_conversions.csproj" />
     <MergedWrapperProjectReference Include="LoopAlignment\LoopsToProcess.csproj" />
     <MergedWrapperProjectReference Include="physicalpromotion\addressinterference.csproj" Condition="'$(_UseOutOfProcessPhysicalPromotionTests)' == 'true'" />
+    <MergedWrapperProjectReference Include="physicalpromotion\CopyPromotedArguments.csproj" />
     <MergedWrapperProjectReference Include="physicalpromotion\fuzzlyn1.csproj" Condition="'$(_UseOutOfProcessPhysicalPromotionTests)' == 'true'" />
     <MergedWrapperProjectReference Include="physicalpromotion\fuzzlyn2.csproj" Condition="'$(_UseOutOfProcessPhysicalPromotionTests)' == 'true'" />
     <MergedWrapperProjectReference Include="physicalpromotion\mixedpromotion.csproj" Condition="'$(_UseOutOfProcessPhysicalPromotionTests)' == 'true'" />

src/tests/JIT/Directed/physicalpromotion/CopyPromotedArguments.cs

@@ -0,0 +1,124 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using Xunit;
+
+public class CopyPromotedArguments
+{
+    private static long s_consumed;
+
+    [Fact]
+    public static void TestEntryPoint()
+    {
+        foreach (long value in new[] { 0L, 1L, -1L, long.MinValue, long.MaxValue })
+        {
+            CopyArgumentIsolation(value);
+            CopyArgumentAcrossException(value);
+            Assert.Equal(value + 1 + (int)(value + 1), CleanSource(value));
+        }
+    }
+
+    [StructLayout(LayoutKind.Explicit, Size = 24)]
+    private struct S
+    {
+        [FieldOffset(0)] public long Wide;
+        [FieldOffset(0)] public int Narrow;
+        [FieldOffset(8)] public long Other;
+        [FieldOffset(16)] public long Last;
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static S Create(long value) => new S { Wide = value, Other = value + 1, Last = value + 2 };
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static int Observe(int value) => value;
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static void Observe(long value) { }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static void Consume(S value)
+    {
+        Assert.Equal(value.Other + 1, value.Last);
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static void CheckFields(S value, long wide, long other, long last)
+    {
+        Assert.Equal(wide, value.Wide);
+        Assert.Equal(other, value.Other);
+        Assert.Equal(last, value.Last);
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static void ConsumeAndModify(S value)
+    {
+        s_consumed = value.Wide;
+        value.Wide = ~value.Wide;
+        value.Other = -7;
+        CheckFields(value, ~s_consumed, -7, s_consumed + 1);
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static void CopyArgumentIsolation(long value)
+    {
+        S src = Create(value);
+        src.Wide++;
+        Observe(src.Wide);
+        Observe(src.Wide);
+        Observe(src.Wide);
+        ConsumeAndModify(src);
+        Assert.Equal(value + 1, s_consumed);
+        CheckFields(src, value + 1, value + 1, value + 2);
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static void ConsumeThenThrow(S value)
+    {
+        ConsumeAndModify(value);
+        throw new System.InvalidOperationException();
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static void CopyArgumentAcrossException(long value)
+    {
+        S src = Create(value);
+        try
+        {
+            src.Wide++;
+            Observe(src.Wide);
+            Observe(src.Wide);
+            Observe(src.Wide);
+            ConsumeThenThrow(src);
+        }
+        catch (System.InvalidOperationException)
+        {
+            CheckFields(src, value + 1, value + 1, value + 2);
+        }
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static long CleanSource(long value)
+    {
+        // X64-WINDOWS: call {{.*}}CopyPromotedArguments:Create
+        // Build the outgoing argument from the replacement, without synchronizing
+        // source storage and immediately loading across that narrow store.
+        // X64-WINDOWS: mov [[VALUE:r[a-z0-9]+]], qword ptr [rsp+[[SOURCE:0x[0-9A-Fa-f]+]]]
+        // X64-WINDOWS-NOT: mov qword ptr [rsp+[[SOURCE]]], [[VALUE]]
+        // X64-WINDOWS: call {{.*}}CopyPromotedArguments:Consume
+        S src = Create(value);
+        src.Wide++;
+        Observe(src.Wide);
+        Observe(src.Wide);
+        Observe(src.Wide);
+        Consume(src); // The outgoing copy can take Wide directly from its replacement.
+        S dst = src;
+        Observe(dst.Narrow);
+        Observe(dst.Narrow);
+        Observe(dst.Narrow);
+        Consume(src);
+        return src.Wide + dst.Narrow;
+    }
+}

src/tests/JIT/Directed/physicalpromotion/CopyPromotedArguments.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <RequiresProcessIsolation>true</RequiresProcessIsolation>
+    <DisableProjectBuild Condition="'$(TargetsMobile)' == 'true' and ('$(TargetsBrowser)' != 'true' or '$(RuntimeFlavor)' != 'coreclr')">true</DisableProjectBuild>
+    <Optimize>True</Optimize>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="CopyPromotedArguments.cs">
+      <HasDisasmCheck>true</HasDisasmCheck>
+    </Compile>
+    <CLRTestEnvironmentVariable Include="DOTNET_TieredCompilation" Value="0" />
+    <CLRTestEnvironmentVariable Include="DOTNET_JITMinOpts" Value="0" />
+  </ItemGroup>
+</Project>