diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index f7d9192cb588ec..b435916a1d2f99 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -441,6 +441,7 @@ class CodeGen final : public CodeGenInterface #if defined(TARGET_ARM64) void genUnknownSizeFrame(); + void genZeroInitializeUnknownSizeFrame(); #endif #elif defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) @@ -467,6 +468,9 @@ class CodeGen final : public CodeGenInterface void genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pInitRegZeroed, regMaskTP maskArgRegsLiveIn); void genPoisonFrame(regMaskTP bbRegLiveIn); +#ifdef TARGET_ARM64 + void genPoisonUnknownSizeVariable(int varNum, char poisonVal); +#endif #if defined(TARGET_ARM) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index ba92d14b234ea4..3603f2990641dd 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -5962,4 +5962,41 @@ BasicBlock* CodeGen::genGetThrowHelper(SpecialCodeKind codeKind) return excpRaisingBlock; } +// +void CodeGen::genPoisonUnknownSizeVariable(int varNum, char poisonVal) +{ + assert(varNum >= 0); + LclVarDsc* varDsc = m_compiler->lvaGetDesc(varNum); + + assert(varDsc->IsAddressExposed()); + + if (varDsc->TypeIs(TYP_SIMD)) + { + // mov z9.b, #poisonVal + GetEmitter()->emitIns_R_I(INS_sve_mov, EA_SCALABLE, REG_SCRATCH_V, (ssize_t)poisonVal, INS_OPTS_SCALABLE_B); + // str z9, [x19, $index MUL VL] + GetEmitter()->emitIns_S_R(INS_sve_str, EA_SCALABLE, REG_SCRATCH_V, varNum, 0); + } + else + { + assert(varDsc->TypeIs(TYP_MASK)); + // At the moment we don't have a single instruction to produce an arbitrary value in a predicate register, + // since it would have to fit some common element masking pattern. We can, however, produce a repeating + // 64-bit value in a vector, then convert this to a predicate using cmpne. + const ssize_t vectorPoisonVal = static_cast(0xffff0000ffff00ffULL); + // mov x9, #vectorPoisonVal + instGen_Set_Reg_To_Imm(EA_8BYTE, REG_SCRATCH, vectorPoisonVal); + // dup z9.d, x9 + GetEmitter()->emitIns_R_R(INS_sve_dup, EA_8BYTE, REG_SCRATCH_V, REG_SCRATCH, INS_OPTS_SCALABLE_D); + // ptrue p4.b + GetEmitter()->emitIns_R_PATTERN(INS_sve_ptrue, EA_SCALABLE, REG_SCRATCH_P, INS_OPTS_SCALABLE_B, + SVE_PATTERN_ALL); + // cmpne p4.b, p4/z, z9.b, #0 + GetEmitter()->emitIns_R_R_R_I(INS_sve_cmpne, EA_SCALABLE, REG_SCRATCH_P, REG_SCRATCH_P, REG_SCRATCH_V, 0, + INS_OPTS_SCALABLE_B); + // str p4, [x19, $index MUL VL] + GetEmitter()->emitIns_S_R(INS_sve_str, EA_SCALABLE, REG_SCRATCH_P, varNum, 0); + } +} + #endif // TARGET_ARM64 diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index cfa2b5fd82bfbb..91527e38610c1d 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -5039,6 +5039,54 @@ void CodeGen::genUnknownSizeFrame() GetEmitter()->emitIns_R_R_R_R(INS_msub, EA_8BYTE, REG_SP, rsvd, REG_SCRATCH, REG_SP); } } + +//---------------------------------------------------------------------------- +// +// genZeroInitializeUnknownSizeFrame: Zero-initialize the UnknownSizeFrame stack space. +// +// Remarks: +// This function emits code that assumes the state of sp has not been modified since +// establishing the UnknownSizeFrame. sp must point to the end of the UnknownSizeFrame. +// +void CodeGen::genZeroInitializeUnknownSizeFrame() +{ + assert(m_compiler->compUsesUnknownSizeFrame); + + unsigned vectorCount = m_compiler->unkSizeFrame.FrameSizeInVectors(); + + assert(vectorCount > 0); + + // z9 <== {0, 0, ...} + GetEmitter()->emitIns_R_I(INS_sve_mov, EA_SCALABLE, REG_SCRATCH_V, 0, INS_OPTS_SCALABLE_B); + + // For small vector counts, emit unrolled loop of vector stores. + // Unrolling to a maximum of 5 stores optimizes for code size rather than performance. + // TODO-SVE: Does unrolling further improve performance? + if (vectorCount <= 5) + { + for (unsigned i = 0; i < vectorCount; i++) + { + // str z9, [sp, #i MUL VL] + GetEmitter()->emitIns_R_R_I(INS_sve_str, EA_SCALABLE, REG_SCRATCH_V, REG_SP, i); + } + } + else + { + // $cursor <== x19 + inst_Mov(TYP_BYREF, REG_SCRATCH, REG_UNKBASE, false); + BasicBlock* loop = genCreateTempLabel(); + // loop: + genDefineInlineTempLabel(loop); + // addvl $cursor, $cursor, #-1 + GetEmitter()->emitIns_R_R_I(INS_sve_addvl, EA_8BYTE, REG_SCRATCH, REG_SCRATCH, -1); + // str z9, [$cursor] + GetEmitter()->emitIns_R_R(INS_sve_str, EA_SCALABLE, REG_SCRATCH_V, REG_SCRATCH); + // cmp sp, $cursor + GetEmitter()->emitIns_R_R(INS_cmp, EA_8BYTE, REG_SP, REG_SCRATCH, INS_OPTS_UXTX); + // b.ne loop + GetEmitter()->emitIns_J(INS_bne, loop); + } +} #endif /***************************************************************************** diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 7f3cdc29802214..26c0dfb8f65470 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -8508,6 +8508,14 @@ void CodeGen::genPoisonFrame(regMaskTP regLiveIn) assert(varDsc->lvOnFrame); +#ifdef TARGET_ARM64 + if (m_compiler->lvaIsUnknownSizeLocal(varNum)) + { + genPoisonUnknownSizeVariable(varNum, (char)poisonVal); + continue; + } +#endif + unsigned int size = m_compiler->lvaLclStackHomeSize(varNum); if ((size / TARGET_POINTER_SIZE) > 16) { diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index 49f67585225024..875b3ae171e816 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -460,6 +460,13 @@ void CodeGen::genCodeForBlock(BasicBlock* block) } #endif +#ifdef TARGET_ARM64 + if (m_compiler->compUsesUnknownSizeFrame && block->IsFirst()) + { + genZeroInitializeUnknownSizeFrame(); + } +#endif + #ifndef TARGET_WASM // TODO-WASM: enable genPoisonFrame // Emit poisoning into the init BB that comes right after prolog. // We cannot emit this code in the prolog as it might use a helper call that kills argument regs. diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 94c1052c055549..ca8b294dd9732b 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -429,7 +429,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) case IF_DR_2A: // DR_2A X..........mmmmm ......nnnnn..... Rn Rm assert(isValidGeneralDatasize(id->idOpSize())); - assert(isGeneralRegister(id->idReg1())); + assert(isGeneralRegisterOrZR(id->idReg1())); assert(isGeneralRegister(id->idReg2())); break; @@ -8252,22 +8252,43 @@ void emitter::emitIns_R_S(instruction ins, emitAttr attr, regNumber reg1, int va isSimple = false; reg2 = REG_UNKBASE; + var_types localType = TYP_UNDEF; + if (varx >= 0) { - imm = m_compiler->unkSizeFrame.GetAddressingOffset(m_compiler->lvaGetDesc(varx)); + LclVarDsc* varDsc = m_compiler->lvaGetDesc(varx); + imm = m_compiler->unkSizeFrame.GetAddressingOffset(varDsc); + localType = varDsc->TypeGet(); } else { - imm = m_compiler->unkSizeFrame.GetAddressingOffset(codeGen->regSet.tmpGetNum(varx)); + TempDsc* tmpDsc = codeGen->regSet.tmpGetNum(varx); + imm = m_compiler->unkSizeFrame.GetAddressingOffset(tmpDsc); + localType = tmpDsc->tdTempType(); } switch (ins) { case INS_lea: - // We shouldn't be materializing the address of a mask. - assert(m_compiler->lvaGetActualType(varx) != TYP_MASK); - // addvl reg1, x19, #imm - emitIns_R_R_I(INS_sve_addvl, EA_8BYTE, reg1, REG_UNKBASE, imm); + // TODO-SVE: Support materializing address of a mask local / temp. + assert(localType != TYP_MASK); + if (isValidSimm<6>(imm)) + { + // addvl reg1, x19, #imm + emitIns_R_R_I(INS_sve_addvl, EA_8BYTE, reg1, REG_UNKBASE, imm); + } + else + { + // Cannot encode immediate, generate `addr = fp + imm * VL`. + // + // set reg1 = imm + // rdvl rsvd, #1 + // madd reg1, reg1, rsvd, x19 + regNumber rsvd = codeGen->rsGetRsvdReg(); + codeGen->instGen_Set_Reg_To_Imm(EA_8BYTE, reg1, imm); + emitIns_R_I(INS_sve_rdvl, EA_8BYTE, rsvd, 1); + emitIns_R_R_R_R(INS_madd, EA_8BYTE, reg1, reg1, rsvd, REG_UNKBASE); + } return; case INS_sve_ldr: diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 4571e903b4215f..6943985ab0e3bf 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -2450,6 +2450,19 @@ void LinearScan::buildIntervals() currentLoc += 2; } +#ifdef TARGET_ARM64 + if (m_compiler->compUsesUnknownSizeFrame && (block == m_compiler->fgFirstBB)) + { + regMaskTP killed = RBM_NONE; + killed.AddRegNumInMask(REG_SCRATCH); + killed.AddRegNumInMask(REG_SCRATCH_V); + killed.AddRegNumInMask(REG_SCRATCH_P); + + addKillForRegs(killed, currentLoc + 1); + currentLoc += 2; + } +#endif + LIR::Range& blockRange = LIR::AsRange(block); for (GenTree* node : blockRange) { diff --git a/src/coreclr/jit/targetarm64.h b/src/coreclr/jit/targetarm64.h index bea112f319f5d6..0f196cacf5e836 100644 --- a/src/coreclr/jit/targetarm64.h +++ b/src/coreclr/jit/targetarm64.h @@ -139,8 +139,10 @@ #define REG_SHIFT REG_NA #define RBM_SHIFT RBM_ALLINT -// This is a general scratch register that does not conflict with the argument registers +// Scratch registers that do not conflict with the argument registers, usually for use in function prolog #define REG_SCRATCH REG_R9 +#define REG_SCRATCH_V REG_V9 +#define REG_SCRATCH_P REG_P4 // This is a general register that can be optionally reserved for other purposes during codegen #define REG_OPT_RSVD REG_IP1