From dd4e33835afe4569a74b4ecfbe01bf49fe7d1623 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kendall=20Gonz=C3=A1lez=20Le=C3=B3n?= Date: Thu, 20 Aug 2026 17:12:51 -0700 Subject: [PATCH 1/6] Disable memory form of NDD instructions by default for optimal performance on latest Intel processors with APX --- src/coreclr/jit/codegenxarch.cpp | 4 +++- src/coreclr/jit/emitxarch.cpp | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 72fa1d1a34e311..8a1e7a9bfe718f 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -1065,7 +1065,9 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode) else { // when reg3 != reg1 && reg3 != reg2, and NDD is available, we can use APX-EVEX.ND to optimize the codegen. - eligibleForNDD = emit->DoJitUseApxNDD(ins); + // On the latest Intel processors with APX, the memory form of NDD instructions has to be turned off by + // default for optimal hardware performance, so keep NDD only for register and immediate sources. + eligibleForNDD = emit->DoJitUseApxNDD(ins) && !op2->isUsedFromMemory(); if (!eligibleForNDD) { var_types op1Type = op1->TypeGet(); diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index e6f46e8b7f702d..3621ee81a1cf39 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -10468,7 +10468,9 @@ regNumber emitter::emitIns_BASE_R_R_RM( regNumber r = REG_NA; assert(regOp->isUsedFromReg()); - bool useApxNdd = DoJitUseApxNDD(ins); + // On the latest Intel processors with APX, the memory form of NDD instructions has to be turned off by default + // for optimal hardware performance, so fall back to the mov+op sequence when the RM source is in memory. + bool useApxNdd = DoJitUseApxNDD(ins) && !rmOp->isUsedFromMemory(); if (emitIns_Mov(INS_mov, attr, targetReg, regOp->GetRegNum(), true, useApxNdd) && useApxNdd) { From 13a07f24c03b943d154d61f899fdfce8c7d974c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kendall=20Gonz=C3=A1lez=20Le=C3=B3n?= Date: Mon, 24 Aug 2026 23:03:48 -0600 Subject: [PATCH 2/6] Adjust commit message Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/coreclr/jit/emitxarch.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index 3621ee81a1cf39..8767016cf059a5 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -10468,8 +10468,8 @@ regNumber emitter::emitIns_BASE_R_R_RM( regNumber r = REG_NA; assert(regOp->isUsedFromReg()); - // On the latest Intel processors with APX, the memory form of NDD instructions has to be turned off by default - // for optimal hardware performance, so fall back to the mov+op sequence when the RM source is in memory. + // Disable the memory-source form of NDD (EVEX.ND) instructions for performance reasons; fall back to the + // mov+op sequence when the RM source is in memory. bool useApxNdd = DoJitUseApxNDD(ins) && !rmOp->isUsedFromMemory(); if (emitIns_Mov(INS_mov, attr, targetReg, regOp->GetRegNum(), true, useApxNdd) && useApxNdd) From 03b0fceeefda40799a0406086663a1140ac2941a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kendall=20Gonz=C3=A1lez=20Le=C3=B3n?= Date: Tue, 25 Aug 2026 13:31:56 -0600 Subject: [PATCH 3/6] Update comment wording about the change Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/coreclr/jit/codegenxarch.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 8a1e7a9bfe718f..682e51ae6832c8 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -1065,8 +1065,7 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode) else { // when reg3 != reg1 && reg3 != reg2, and NDD is available, we can use APX-EVEX.ND to optimize the codegen. - // On the latest Intel processors with APX, the memory form of NDD instructions has to be turned off by - // default for optimal hardware performance, so keep NDD only for register and immediate sources. + // For performance, avoid the memory-source form of EVEX.ND; keep NDD only for register and immediate sources. eligibleForNDD = emit->DoJitUseApxNDD(ins) && !op2->isUsedFromMemory(); if (!eligibleForNDD) { From d7ec75526a8ffcd18766381788250612b4cd149d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kendall=20Gonz=C3=A1lez=20Le=C3=B3n?= Date: Mon, 5 Oct 2026 14:01:17 -0700 Subject: [PATCH 4/6] JIT: Decide APX NDD memory-source eligibility in CodeGen The memory-source form of APX NDD (`ins dst, src1, [mem]`) is not profitable, so it is never selected; the legacy `mov dst, src1` + `ins dst, [mem]` pair is used instead. Register and immediate NDD forms are unchanged. Move the decision out of emitIns_BASE_R_R_RM into its two callers, genCodeForBinary and genCodeForMul, through a new DoJitUseApxNDD(ins, rmOp) overload, and pass it in explicitly. The emitter asserts the caller's choice, and that the fallback `mov` cannot overwrite an address register of the memory operand. LSRA guarantees the latter by modelling these nodes as read-modify-write. No codegen change when APX NDD is disabled (the default). --- src/coreclr/jit/codegenxarch.cpp | 9 ++--- src/coreclr/jit/emitxarch.cpp | 57 +++++++++++++++++++++++++++----- src/coreclr/jit/emitxarch.h | 10 ++++-- 3 files changed, 62 insertions(+), 14 deletions(-) diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index cc79f06002ba85..cea71a72bb169c 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -1073,11 +1073,12 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode) else { // when reg3 != reg1 && reg3 != reg2, and NDD is available, we can use APX-EVEX.ND to optimize the codegen. - // For performance, avoid the memory-source form of EVEX.ND; keep NDD only for register and immediate sources. - eligibleForNDD = emit->DoJitUseApxNDD(ins) && !op2->isUsedFromMemory(); + eligibleForNDD = emit->DoJitUseApxNDD(ins, op2); if (!eligibleForNDD) { var_types op1Type = op1->TypeGet(); + // The mov must not clobber a base/index register of a contained op2; LSRA delay-frees those. + assert((op2->gtGetContainedRegMask() & genRegMask(targetReg)) == 0); inst_Mov(op1Type, targetReg, op1reg, /* canSkip */ false); regSet.verifyRegUsed(targetReg); gcInfo.gcMarkRegPtrVal(targetReg, op1Type); @@ -1118,7 +1119,7 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode) assert(dst->isUsedFromReg()); assert(op1reg != targetReg); assert(op2reg != targetReg); - r = emit->emitIns_BASE_R_R_RM(ins, emitTypeSize(treeNode), targetReg, treeNode, dst, src); + r = emit->emitIns_BASE_R_R_RM(ins, emitTypeSize(treeNode), targetReg, treeNode, dst, src, eligibleForNDD); } else { @@ -1277,7 +1278,7 @@ void CodeGen::genCodeForMul(GenTreeOp* treeNode) } assert(regOp->isUsedFromReg()); - emit->emitIns_BASE_R_R_RM(ins, size, mulTargetReg, treeNode, regOp, rmOp); + emit->emitIns_BASE_R_R_RM(ins, size, mulTargetReg, treeNode, regOp, rmOp, emit->DoJitUseApxNDD(ins, rmOp)); // Move the result to the desired register, if necessary if (ins == INS_mulEAX) diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index 9b5bdc67472166..bc56cb5bed3eab 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -1028,6 +1028,26 @@ bool emitter::DoJitUseApxNDD(instruction ins) const #endif } +//------------------------------------------------------------------------ +// DoJitUseApxNDD: Answer the question: does JIT use APX NDD feature on the given instruction, +// given the r/m operand it would encode? +// +// Arguments: +// ins - instruction to test +// rmOp - the operand that would be encoded as the r/m source +// +// Return Value: +// true if JIT allows APX NDD to be applied on the instruction. +// +bool emitter::DoJitUseApxNDD(instruction ins, GenTree* rmOp) const +{ +#if !defined(TARGET_AMD64) + return false; +#else + return DoJitUseApxNDD(ins) && !rmOp->isUsedFromMemory(); +#endif +} + inline bool emitter::IsApxConditionalInstruction(instruction ins) { #ifdef TARGET_AMD64 @@ -10467,16 +10487,38 @@ void emitter::emitIns_BASE_R_R_I(instruction ins, emitAttr attr, regNumber op1Re } } -regNumber emitter::emitIns_BASE_R_R_RM( - instruction ins, emitAttr attr, regNumber targetReg, GenTree* treeNode, GenTree* regOp, GenTree* rmOp) +//------------------------------------------------------------------------ +// emitIns_BASE_R_R_RM: Emit a binary instruction with a register and an r/m source into targetReg. +// +// Arguments: +// ins - the instruction to emit +// attr - the instruction operand size +// targetReg - the destination register +// treeNode - the node being generated +// regOp - the operand that is in a register +// rmOp - the operand that may be contained (register, memory or immediate) +// useApxNdd - true to emit the non-destructive `ins targetReg, regOp, rmOp` form +// +regNumber emitter::emitIns_BASE_R_R_RM(instruction ins, + emitAttr attr, + regNumber targetReg, + GenTree* treeNode, + GenTree* regOp, + GenTree* rmOp, + bool useApxNdd) { - bool requiresOverflowCheck = treeNode->gtOverflowEx(); - regNumber r = REG_NA; assert(regOp->isUsedFromReg()); + assert(!useApxNdd || DoJitUseApxNDD(ins, rmOp)); - // Disable the memory-source form of NDD (EVEX.ND) instructions for performance reasons; fall back to the - // mov+op sequence when the RM source is in memory. - bool useApxNdd = DoJitUseApxNDD(ins) && !rmOp->isUsedFromMemory(); +#ifdef DEBUG + if (!useApxNdd && (targetReg != regOp->GetRegNum())) + { + // The `mov` below must not clobber a base/index register of rmOp. LSRA guarantees this: + // isRMWRegOper reports RMW for these nodes, so BuildRMWUses routes the contained operand + // through BuildDelayFreeUses, which keeps its address registers live past the def. + assert((rmOp->gtGetContainedRegMask() & genRegMask(targetReg)) == 0); + } +#endif // DEBUG if (emitIns_Mov(INS_mov, attr, targetReg, regOp->GetRegNum(), true, useApxNdd) && useApxNdd) { @@ -10484,7 +10526,6 @@ regNumber emitter::emitIns_BASE_R_R_RM( } return emitInsBinary(ins, attr, treeNode, rmOp); - ; } //---------------------------------------------------------------------------------------- diff --git a/src/coreclr/jit/emitxarch.h b/src/coreclr/jit/emitxarch.h index 81274a8bb2af41..696ac752fffc72 100644 --- a/src/coreclr/jit/emitxarch.h +++ b/src/coreclr/jit/emitxarch.h @@ -145,6 +145,7 @@ static bool IsBitTestInstruction(instruction ins); bool IsLegacyMap1(code_t code) const; bool IsSimdVexOrEvexEncodableInstruction(instruction ins) const; bool DoJitUseApxNDD(instruction ins) const; +bool DoJitUseApxNDD(instruction ins, GenTree* rmOp) const; code_t insEncodeMIreg(const instrDesc* id, regNumber reg, emitAttr size, code_t code); @@ -1246,8 +1247,13 @@ void emitIns_BASE_R_R(instruction ins, emitAttr attr, regNumber op1Reg, regNumbe void emitIns_BASE_R_R_I(instruction ins, emitAttr attr, regNumber op1Reg, regNumber op2Reg, int ival); -regNumber emitIns_BASE_R_R_RM( - instruction ins, emitAttr attr, regNumber targetReg, GenTree* treeNode, GenTree* regOp, GenTree* rmOp); +regNumber emitIns_BASE_R_R_RM(instruction ins, + emitAttr attr, + regNumber targetReg, + GenTree* treeNode, + GenTree* regOp, + GenTree* rmOp, + bool useApxNdd); #ifdef TARGET_AMD64 // Is the last instruction emitted a call instruction? From 18a58178fd049c562362f67a9f71a45b196e241c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kendall=20Gonz=C3=A1lez=20Le=C3=B3n?= Date: Mon, 5 Oct 2026 14:01:18 -0700 Subject: [PATCH 5/6] JIT: Let LSRA skip the op2 delay-free for APX NDD sub For read-modify-write nodes, BuildRMWUses marks op2 delay-free because the legacy sequence writes the destination before reading op2. The APX NDD form reads both sources first, so the constraint is not needed when codegen emits NDD. For an integer sub with a register op2, skip the delay-free when NDD is enabled for sub and op1 is a register-candidate local that stays live after the node. The destination can't take op1's register there, so the JIT already emitted the three-register NDD form; now the destination may take op2's register. Codegen emits NDD `sub dst, op1, op2` with dst == op2, and the emitter allows it. The op1 preference is unchanged, and a contained op2 keeps its delay-free. No codegen change when APX NDD is disabled (the default). --- src/coreclr/jit/codegenxarch.cpp | 18 +++++++++++++----- src/coreclr/jit/emitxarch.cpp | 7 ++----- src/coreclr/jit/lsraxarch.cpp | 7 +++++++ 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index cea71a72bb169c..3a796297e1581d 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -1043,10 +1043,19 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode) // be the same local. else if (op2reg == targetReg) { - assert(GenTree::OperIsCommutative(oper) || genIsSameLocalVar(op1, op2)); - - dst = op2; - src = op1; + if (GenTree::OperIsCommutative(oper) || genIsSameLocalVar(op1, op2)) + { + dst = op2; + src = op1; + } + else + { + // LSRA only lets a non-commutative op reuse op2's register when it is emitted as APX NDD. + eligibleForNDD = emit->DoJitUseApxNDD(ins, op2); + assert(eligibleForNDD); + dst = op1; + src = op2; + } } // now we know there are 3 different operands so attempt to use LEA else if (oper == GT_ADD && !varTypeIsFloating(treeNode) && !treeNode->gtOverflowEx() // LEA does not set flags @@ -1118,7 +1127,6 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode) // operands should be already formatted above assert(dst->isUsedFromReg()); assert(op1reg != targetReg); - assert(op2reg != targetReg); r = emit->emitIns_BASE_R_R_RM(ins, emitTypeSize(treeNode), targetReg, treeNode, dst, src, eligibleForNDD); } else diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index bc56cb5bed3eab..b5de597794d54d 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -6529,12 +6529,9 @@ regNumber emitter::emitInsBinary(instruction ins, emitAttr attr, GenTree* dst, G assert(IsApxNddEncodableInstruction(ins)); // targetReg has to be an actual register if using NDD. assert(targetReg < REG_STK); - // make sure target register is not either of the src registers. + // targetReg may be src's register (a non-commutative sub), but never dst's. assert(dst->isUsedFromReg()); - regNumber dstreg = dst->GetRegNum(); - regNumber srcreg = src->isUsedFromReg() ? src->GetRegNum() : REG_NA; - assert(targetReg != dstreg); - assert(targetReg != srcreg); + assert(targetReg != dst->GetRegNum()); } #endif diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index 4b170bd03912f6..01b913229f3e92 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -888,6 +888,13 @@ int LinearScan::BuildRMWUses( { delayUseOperand = nullptr; } + else if (node->OperIs(GT_SUB) && !varTypeIsFloating(node) && !op2->isContained() && op1->OperIs(GT_LCL_VAR) && + isCandidateLocalRef(op1) && !op1->AsLclVar()->IsLastUse(0) && + m_compiler->GetEmitter()->DoJitUseApxNDD(INS_sub)) + { + // op1 stays live, so dst can't take its register; APX NDD "sub dst, op1, op2" lets dst take op2's instead. + delayUseOperand = nullptr; + } if (delayUseOperand != nullptr) { assert(!prefOp1 || delayUseOperand != op1); From 786332c6c5aaf700a11403693b5c1d10ad3af59e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kendall=20Gonz=C3=A1lez=20Le=C3=B3n?= Date: Mon, 5 Oct 2026 19:29:40 -0700 Subject: [PATCH 6/6] JIT: Address review feedback for APX NDD sub Move the LSRA check that drops the op2 delay-free for APX NDD sub out of the else-if chain into a separate check under TARGET_AMD64, keyed on delayUseOperand. The condition is unchanged. In genCodeForBinary, let a non-commutative op whose destination got op2's register fall through to the three-register path, which already emits NDD. On the non-NDD path, a noway_assert keeps release builds from emitting a mov that would overwrite op2 if LSRA and codegen ever disagree. No codegen change. --- src/coreclr/jit/codegenxarch.cpp | 21 ++++++--------------- src/coreclr/jit/lsraxarch.cpp | 12 ++++++++---- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 3a796297e1581d..b0b6488117d714 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -1041,21 +1041,10 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode) // we can convert it into reg1 = reg1 op reg2 and emit // the same code as above. Or we need both operands to // be the same local. - else if (op2reg == targetReg) + else if ((op2reg == targetReg) && (GenTree::OperIsCommutative(oper) || genIsSameLocalVar(op1, op2))) { - if (GenTree::OperIsCommutative(oper) || genIsSameLocalVar(op1, op2)) - { - dst = op2; - src = op1; - } - else - { - // LSRA only lets a non-commutative op reuse op2's register when it is emitted as APX NDD. - eligibleForNDD = emit->DoJitUseApxNDD(ins, op2); - assert(eligibleForNDD); - dst = op1; - src = op2; - } + dst = op2; + src = op1; } // now we know there are 3 different operands so attempt to use LEA else if (oper == GT_ADD && !varTypeIsFloating(treeNode) && !treeNode->gtOverflowEx() // LEA does not set flags @@ -1082,11 +1071,13 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode) else { // when reg3 != reg1 && reg3 != reg2, and NDD is available, we can use APX-EVEX.ND to optimize the codegen. + // LSRA also allows reg3 == reg2 for a non-commutative op, but only when it is emitted as NDD. eligibleForNDD = emit->DoJitUseApxNDD(ins, op2); if (!eligibleForNDD) { var_types op1Type = op1->TypeGet(); - // The mov must not clobber a base/index register of a contained op2; LSRA delay-frees those. + // The mov must not clobber op2: its register, or a base/index register of a contained op2. + noway_assert(op2reg != targetReg); assert((op2->gtGetContainedRegMask() & genRegMask(targetReg)) == 0); inst_Mov(op1Type, targetReg, op1reg, /* canSkip */ false); regSet.verifyRegUsed(targetReg); diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index 01b913229f3e92..7f0dc9d5263fd3 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -888,13 +888,17 @@ int LinearScan::BuildRMWUses( { delayUseOperand = nullptr; } - else if (node->OperIs(GT_SUB) && !varTypeIsFloating(node) && !op2->isContained() && op1->OperIs(GT_LCL_VAR) && - isCandidateLocalRef(op1) && !op1->AsLclVar()->IsLastUse(0) && - m_compiler->GetEmitter()->DoJitUseApxNDD(INS_sub)) + +#ifdef TARGET_AMD64 + if ((delayUseOperand != nullptr) && node->OperIs(GT_SUB) && !varTypeIsFloating(node) && !op2->isContained() && + m_compiler->GetEmitter()->DoJitUseApxNDD(INS_sub) && op1->OperIs(GT_LCL_VAR) && isCandidateLocalRef(op1) && + !op1->AsLclVar()->IsLastUse(0)) { - // op1 stays live, so dst can't take its register; APX NDD "sub dst, op1, op2" lets dst take op2's instead. + // NDD reads op2 before writing dst. Not when op1 dies: dst should reuse op1's register (legacy sub). delayUseOperand = nullptr; } +#endif // TARGET_AMD64 + if (delayUseOperand != nullptr) { assert(!prefOp1 || delayUseOperand != op1);