diff --git a/src/coreclr/src/jit/codegen.h b/src/coreclr/src/jit/codegen.h index d6b53ef475d7a..5c8a04c466d12 100644 --- a/src/coreclr/src/jit/codegen.h +++ b/src/coreclr/src/jit/codegen.h @@ -1117,7 +1117,9 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX void genUnspillLocal( unsigned varNum, var_types type, GenTreeLclVar* lclNode, regNumber regNum, bool reSpill, bool isLastUse); void genUnspillRegIfNeeded(GenTree* tree); + void genUnspillRegIfNeeded(GenTree* tree, unsigned multiRegIndex); regNumber genConsumeReg(GenTree* tree); + regNumber genConsumeReg(GenTree* tree, unsigned multiRegIndex); void genCopyRegIfNeeded(GenTree* tree, regNumber needReg); void genConsumeRegAndCopy(GenTree* tree, regNumber needReg); @@ -1130,6 +1132,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX } void genRegCopy(GenTree* tree); + regNumber genRegCopy(GenTree* tree, unsigned multiRegIndex); void genTransferRegGCState(regNumber dst, regNumber src); void genConsumeAddress(GenTree* addr); void genConsumeAddrMode(GenTreeAddrMode* mode); @@ -1278,7 +1281,8 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX void genEHFinallyOrFilterRet(BasicBlock* block); #endif // !FEATURE_EH_FUNCLETS - void genMultiRegStoreToLocal(GenTree* treeNode); + void genMultiRegStoreToSIMDLocal(GenTreeLclVar* lclNode); + void genMultiRegStoreToLocal(GenTreeLclVar* lclNode); // Codegen for multi-register struct returns. bool isStructReturn(GenTree* treeNode); diff --git a/src/coreclr/src/jit/codegenarm.cpp b/src/coreclr/src/jit/codegenarm.cpp index 015ac03a68b01..7acdf3fbab219 100644 --- a/src/coreclr/src/jit/codegenarm.cpp +++ b/src/coreclr/src/jit/codegenarm.cpp @@ -1048,7 +1048,7 @@ void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* tree) // var = call, where call returns a multi-reg return value // case is handled separately. - if (data->gtSkipReloadOrCopy()->IsMultiRegCall()) + if (data->gtSkipReloadOrCopy()->IsMultiRegNode()) { genMultiRegStoreToLocal(tree); } diff --git a/src/coreclr/src/jit/codegenarm64.cpp b/src/coreclr/src/jit/codegenarm64.cpp index 91b5885f17463..949f7cc605f6e 100644 --- a/src/coreclr/src/jit/codegenarm64.cpp +++ b/src/coreclr/src/jit/codegenarm64.cpp @@ -1919,33 +1919,49 @@ void CodeGen::genCodeForStoreLclFld(GenTreeLclFld* tree) // genCodeForStoreLclVar: Produce code for a GT_STORE_LCL_VAR node. // // Arguments: -// tree - the GT_STORE_LCL_VAR node +// lclNode - the GT_STORE_LCL_VAR node // -void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* tree) +void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* lclNode) { - GenTree* data = tree->gtOp1; + GenTree* data = lclNode->gtOp1; + + // Stores from a multi-reg source are handled separately. + if (data->gtSkipReloadOrCopy()->IsMultiRegNode()) + { + genMultiRegStoreToLocal(lclNode); + return; + } - // var = call, where call returns a multi-reg return value - // case is handled separately. - if (data->gtSkipReloadOrCopy()->IsMultiRegCall()) + LclVarDsc* varDsc = compiler->lvaGetDesc(lclNode); + if (lclNode->IsMultiReg()) { - genMultiRegStoreToLocal(tree); + // This is the case of storing to a multi-reg HFA local from a fixed-size SIMD type. + assert(varTypeIsSIMD(data) && varDsc->lvIsHfa() && (varDsc->GetHfaType() == TYP_FLOAT)); + regNumber operandReg = genConsumeReg(data); + unsigned int regCount = varDsc->lvFieldCnt; + for (unsigned i = 0; i < regCount; ++i) + { + regNumber varReg = lclNode->GetRegByIndex(i); + assert(varReg != REG_NA); + unsigned fieldLclNum = varDsc->lvFieldLclStart + i; + LclVarDsc* fieldVarDsc = compiler->lvaGetDesc(fieldLclNum); + assert(fieldVarDsc->TypeGet() == TYP_FLOAT); + GetEmitter()->emitIns_R_R_I(INS_dup, emitTypeSize(TYP_FLOAT), varReg, operandReg, i); + } } else { - regNumber targetReg = tree->GetRegNum(); + regNumber targetReg = lclNode->GetRegNum(); emitter* emit = GetEmitter(); - unsigned varNum = tree->GetLclNum(); - assert(varNum < compiler->lvaCount); - LclVarDsc* varDsc = compiler->lvaGetDesc(varNum); - var_types targetType = varDsc->GetRegisterType(tree); + unsigned varNum = lclNode->GetLclNum(); + var_types targetType = varDsc->GetRegisterType(lclNode); #ifdef FEATURE_SIMD // storing of TYP_SIMD12 (i.e. Vector3) field if (targetType == TYP_SIMD12) { - genStoreLclTypeSIMD12(tree); + genStoreLclTypeSIMD12(lclNode); return; } #endif // FEATURE_SIMD @@ -1963,7 +1979,7 @@ void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* tree) if (targetReg != REG_NA) { emit->emitIns_R_I(INS_movi, emitActualTypeSize(targetType), targetReg, 0x00, INS_OPTS_16B); - genProduceReg(tree); + genProduceReg(lclNode); } else { @@ -1976,7 +1992,7 @@ void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* tree) assert(targetType == TYP_SIMD8); GetEmitter()->emitIns_S_R(INS_str, EA_8BYTE, REG_ZR, varNum, 0); } - genUpdateLife(tree); + genUpdateLife(lclNode); } return; } @@ -1992,14 +2008,14 @@ void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* tree) if (targetReg == REG_NA) // store into stack based LclVar { - inst_set_SV_var(tree); + inst_set_SV_var(lclNode); instruction ins = ins_Store(targetType); emitAttr attr = emitActualTypeSize(targetType); emit->emitIns_S_R(ins, attr, dataReg, varNum, /* offset */ 0); - genUpdateLife(tree); + genUpdateLife(lclNode); varDsc->SetRegNum(REG_STK); } @@ -2010,7 +2026,7 @@ void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* tree) // Assign into targetReg when dataReg (from op1) is not the same register inst_RV_RV(ins_Copy(targetType), targetReg, dataReg, targetType); } - genProduceReg(tree); + genProduceReg(lclNode); } } } diff --git a/src/coreclr/src/jit/codegenarmarch.cpp b/src/coreclr/src/jit/codegenarmarch.cpp index 9ca28936f24bf..7f7940a903172 100644 --- a/src/coreclr/src/jit/codegenarmarch.cpp +++ b/src/coreclr/src/jit/codegenarmarch.cpp @@ -1354,116 +1354,69 @@ void CodeGen::genPutArgSplit(GenTreePutArgSplit* treeNode) } #endif // FEATURE_ARG_SPLIT +#ifdef FEATURE_SIMD //---------------------------------------------------------------------------------- -// genMultiRegStoreToLocal: store multi-reg return value of a call node to a local +// genMultiRegStoreToSIMDLocal: store multi-reg value to a single-reg SIMD local // // Arguments: -// treeNode - Gentree of GT_STORE_LCL_VAR +// lclNode - GentreeLclVar of GT_STORE_LCL_VAR // // Return Value: // None // -// Assumption: -// The child of store is a multi-reg node. -// -void CodeGen::genMultiRegStoreToLocal(GenTree* treeNode) +void CodeGen::genMultiRegStoreToSIMDLocal(GenTreeLclVar* lclNode) { - assert(treeNode->OperGet() == GT_STORE_LCL_VAR); - assert(varTypeIsStruct(treeNode) || varTypeIsMultiReg(treeNode)); - GenTree* op1 = treeNode->gtGetOp1(); - GenTree* actualOp1 = op1->gtSkipReloadOrCopy(); + regNumber dst = lclNode->GetRegNum(); + GenTree* op1 = lclNode->gtGetOp1(); + GenTree* actualOp1 = op1->gtSkipReloadOrCopy(); + unsigned regCount = + actualOp1->IsMultiRegLclVar() ? actualOp1->AsLclVar()->GetFieldCount(compiler) : actualOp1->GetMultiRegCount(); assert(op1->IsMultiRegNode()); - unsigned regCount = actualOp1->GetMultiRegCount(); - - // Assumption: current implementation requires that a multi-reg - // var in 'var = call' is flagged as lvIsMultiRegRet to prevent it from - // being promoted. - unsigned lclNum = treeNode->AsLclVarCommon()->GetLclNum(); - LclVarDsc* varDsc = compiler->lvaGetDesc(lclNum); - if (op1->OperIs(GT_CALL)) - { - assert(regCount <= MAX_RET_REG_COUNT); - noway_assert(varDsc->lvIsMultiRegRet); - } - genConsumeRegs(op1); - int offset = 0; - - // Check for the case of an enregistered SIMD type that's returned in multiple registers. - if (varDsc->lvIsRegCandidate() && treeNode->GetRegNum() != REG_NA) + // Treat dst register as a homogenous vector with element size equal to the src size + // Insert pieces in reverse order + for (int i = regCount - 1; i >= 0; --i) { - assert(varTypeIsSIMD(treeNode)); - assert(regCount != 0); - - regNumber dst = treeNode->GetRegNum(); - - // Treat dst register as a homogenous vector with element size equal to the src size - // Insert pieces in reverse order - for (int i = regCount - 1; i >= 0; --i) + var_types type = op1->gtSkipReloadOrCopy()->GetRegTypeByIndex(i); + regNumber reg = op1->GetRegByIndex(i); + if (op1->IsCopyOrReload()) { - var_types type = op1->gtSkipReloadOrCopy()->GetRegTypeByIndex(i); - regNumber reg = op1->GetRegByIndex(i); - if (op1->IsCopyOrReload()) - { - // GT_COPY/GT_RELOAD will have valid reg for those positions - // that need to be copied or reloaded. - regNumber reloadReg = op1->AsCopyOrReload()->GetRegNumByIdx(i); - if (reloadReg != REG_NA) - { - reg = reloadReg; - } - } - - assert(reg != REG_NA); - if (varTypeIsFloating(type)) + // GT_COPY/GT_RELOAD will have valid reg for those positions + // that need to be copied or reloaded. + regNumber reloadReg = op1->AsCopyOrReload()->GetRegNumByIdx(i); + if (reloadReg != REG_NA) { - // If the register piece was passed in a floating point register - // Use a vector mov element instruction - // src is not a vector, so it is in the first element reg[0] - // mov dst[i], reg[0] - // This effectively moves from `reg[0]` to `dst[i]`, leaving other dst bits unchanged till further - // iterations - // For the case where reg == dst, if we iterate so that we write dst[0] last, we eliminate the need for - // a temporary - GetEmitter()->emitIns_R_R_I_I(INS_mov, emitTypeSize(type), dst, reg, i, 0); - } - else - { - // If the register piece was passed in an integer register - // Use a vector mov from general purpose register instruction - // mov dst[i], reg - // This effectively moves from `reg` to `dst[i]` - GetEmitter()->emitIns_R_R_I(INS_mov, emitTypeSize(type), dst, reg, i); + reg = reloadReg; } } - genProduceReg(treeNode); - } - else - { - for (unsigned i = 0; i < regCount; ++i) + assert(reg != REG_NA); + if (varTypeIsFloating(type)) { - var_types type = actualOp1->GetRegTypeByIndex(i); - regNumber reg = op1->GetRegByIndex(i); - if (reg == REG_NA) - { - // GT_COPY/GT_RELOAD will have valid reg only for those positions - // that need to be copied or reloaded. - assert(op1->IsCopyOrReload()); - reg = actualOp1->GetRegByIndex(i); - } - - assert(reg != REG_NA); - GetEmitter()->emitIns_S_R(ins_Store(type), emitTypeSize(type), reg, lclNum, offset); - offset += genTypeSize(type); + // If the register piece was passed in a floating point register + // Use a vector mov element instruction + // src is not a vector, so it is in the first element reg[0] + // mov dst[i], reg[0] + // This effectively moves from `reg[0]` to `dst[i]`, leaving other dst bits unchanged till further + // iterations + // For the case where reg == dst, if we iterate so that we write dst[0] last, we eliminate the need for + // a temporary + GetEmitter()->emitIns_R_R_I_I(INS_mov, emitTypeSize(type), dst, reg, i, 0); + } + else + { + // If the register piece was passed in an integer register + // Use a vector mov from general purpose register instruction + // mov dst[i], reg + // This effectively moves from `reg` to `dst[i]` + GetEmitter()->emitIns_R_R_I(INS_mov, emitTypeSize(type), dst, reg, i); } - - // Update variable liveness. - genUpdateLife(treeNode); - varDsc->SetRegNum(REG_STK); } + + genProduceReg(lclNode); } +#endif // FEATURE_SIMD //------------------------------------------------------------------------ // genRangeCheck: generate code for GT_ARR_BOUNDS_CHECK node. diff --git a/src/coreclr/src/jit/codegencommon.cpp b/src/coreclr/src/jit/codegencommon.cpp index b6285d940213c..0449e3291b09c 100644 --- a/src/coreclr/src/jit/codegencommon.cpp +++ b/src/coreclr/src/jit/codegencommon.cpp @@ -11660,6 +11660,7 @@ void CodeGen::genStructReturn(GenTree* treeNode) { varDsc = compiler->lvaGetDesc(actualOp1->AsLclVar()->GetLclNum()); retTypeDesc.InitializeStructReturnType(compiler, varDsc->lvVerTypeInfo.GetClassHandle()); + assert(varDsc->lvIsMultiRegRet); } else { @@ -11670,15 +11671,16 @@ void CodeGen::genStructReturn(GenTree* treeNode) assert(regCount <= MAX_RET_REG_COUNT); #if FEATURE_MULTIREG_RET - if (actualOp1->OperIs(GT_LCL_VAR) && (varTypeIsEnregisterable(op1))) + if (genIsRegCandidateLocal(actualOp1)) { // Right now the only enregisterable structs supported are SIMD vector types. assert(varTypeIsSIMD(op1)); + assert(!actualOp1->AsLclVar()->IsMultiReg()); #ifdef FEATURE_SIMD genSIMDSplitReturn(op1, &retTypeDesc); #endif // FEATURE_SIMD } - else if (actualOp1->OperIs(GT_LCL_VAR)) + else if (actualOp1->OperIs(GT_LCL_VAR) && !actualOp1->AsLclVar()->IsMultiReg()) { GenTreeLclVar* lclNode = actualOp1->AsLclVar(); LclVarDsc* varDsc = compiler->lvaGetDesc(lclNode->GetLclNum()); @@ -11694,20 +11696,36 @@ void CodeGen::genStructReturn(GenTree* treeNode) } else { - assert(actualOp1->IsMultiRegCall()); for (unsigned i = 0; i < regCount; ++i) { var_types type = retTypeDesc.GetReturnRegType(i); regNumber toReg = retTypeDesc.GetABIReturnReg(i); regNumber fromReg = op1->GetRegByIndex(i); - if (fromReg == REG_NA) + if ((fromReg == REG_NA) && op1->OperIs(GT_COPY)) { - assert(op1->IsCopyOrReload()); + // A copy that doesn't copy this field will have REG_NA. + // TODO-Cleanup: It would probably be better to always have a valid reg + // on a GT_COPY, unless the operand is actually spilled. Then we wouldn't have + // to check for this case (though we'd have to check in the genRegCopy that the + // reg is valid). fromReg = actualOp1->GetRegByIndex(i); } - if (fromReg != toReg) + if (fromReg == REG_NA) + { + // This is a spilled field of a multi-reg lclVar. + // We currently only mark a lclVar operand as RegOptional, since we don't have a way + // to mark a multi-reg tree node as used from spill (GTF_NOREG_AT_USE) on a per-reg basis. + assert(varDsc != nullptr); + assert(varDsc->lvPromoted); + unsigned fieldVarNum = varDsc->lvFieldLclStart + i; + assert(compiler->lvaGetDesc(fieldVarNum)->lvOnFrame); + GetEmitter()->emitIns_R_S(ins_Load(type), emitTypeSize(type), toReg, fieldVarNum, 0); + } + else if (fromReg != toReg) { - inst_RV_RV(ins_Copy(type), toReg, fromReg, type); + // Note that ins_Copy(fromReg, type) will return the appropriate register to copy + // between register files if needed. + inst_RV_RV(ins_Copy(fromReg, type), toReg, fromReg, type); } } } @@ -11716,6 +11734,150 @@ void CodeGen::genStructReturn(GenTree* treeNode) #endif } +//---------------------------------------------------------------------------------- +// genMultiRegStoreToLocal: store multi-reg value to a local +// +// Arguments: +// lclNode - Gentree of GT_STORE_LCL_VAR +// +// Return Value: +// None +// +// Assumption: +// The child of store is a multi-reg node. +// +void CodeGen::genMultiRegStoreToLocal(GenTreeLclVar* lclNode) +{ + assert(lclNode->OperIs(GT_STORE_LCL_VAR)); + assert(varTypeIsStruct(lclNode) || varTypeIsMultiReg(lclNode)); + GenTree* op1 = lclNode->gtGetOp1(); + GenTree* actualOp1 = op1->gtSkipReloadOrCopy(); + assert(op1->IsMultiRegNode()); + unsigned regCount = + actualOp1->IsMultiRegLclVar() ? actualOp1->AsLclVar()->GetFieldCount(compiler) : actualOp1->GetMultiRegCount(); + + // Assumption: current implementation requires that a multi-reg + // var in 'var = call' is flagged as lvIsMultiRegRet to prevent it from + // being promoted, unless compiler->lvaEnregMultiRegVars is true. + + unsigned lclNum = lclNode->AsLclVarCommon()->GetLclNum(); + LclVarDsc* varDsc = compiler->lvaGetDesc(lclNum); + if (op1->OperIs(GT_CALL)) + { + assert(regCount <= MAX_RET_REG_COUNT); + noway_assert(varDsc->lvIsMultiRegRet); + } + +#ifdef FEATURE_SIMD + // Check for the case of an enregistered SIMD type that's returned in multiple registers. + if (varDsc->lvIsRegCandidate() && lclNode->GetRegNum() != REG_NA) + { + assert(varTypeIsSIMD(lclNode)); + genMultiRegStoreToSIMDLocal(lclNode); + return; + } +#endif // FEATURE_SIMD + + // We have either a multi-reg local or a local with multiple fields in memory. + // + // The liveness model is as follows: + // use reg #0 from src, including any reload or copy + // define reg #0 + // use reg #1 from src, including any reload or copy + // define reg #1 + // etc. + // Imagine the following scenario: + // There are 3 registers used. Prior to this node, they occupy registers r3, r2 and r1. + // There are 3 registers defined by this node. They need to be placed in r1, r2 and r3, + // in that order. + // + // If we defined the as using all the source registers at once, we'd have to adopt one + // of the following models: + // - All (or all but one) of the incoming sources are marked "delayFree" so that they won't + // get the same register as any of the registers being defined. This would result in copies for + // the common case where the source and destination registers are the same (e.g. when a CALL + // result is assigned to a lclVar, which is then returned). + // - For our example (and for many/most cases) we would have to copy or spill all sources. + // - We allow circular dependencies between source and destination registers. This would require + // the code generator to determine the order in which the copies must be generated, and would + // require a temp register in case a swap is required. This complexity would have to be handled + // in both the normal code generation case, as well as for copies & reloads, as they are currently + // modeled by the register allocator to happen just prior to the use. + // - For our example, a temp would be required to swap r1 and r3, unless a swap instruction is + // available on the target. + // + // By having a multi-reg local use and define each field in order, we avoid these issues, and the + // register allocator will ensure that any conflicts are resolved via spill or inserted COPYs. + // For our example, the register allocator would simple spill r1 because the first def requires it. + // The code generator would move r3 to r1, leave r2 alone, and then load the spilled value into r3. + + int offset = 0; + bool isMultiRegVar = lclNode->IsMultiRegLclVar(); + bool hasRegs = false; + + if (isMultiRegVar) + { + assert(compiler->lvaEnregMultiRegVars); + assert(regCount == varDsc->lvFieldCnt); + } + for (unsigned i = 0; i < regCount; ++i) + { + regNumber reg = genConsumeReg(op1, i); + var_types type = actualOp1->GetRegTypeByIndex(i); + // genConsumeReg will return the valid register, either from the COPY + // or from the original source. + assert(reg != REG_NA); + regNumber varReg = REG_NA; + if (isMultiRegVar) + { + regNumber varReg = lclNode->GetRegByIndex(i); + unsigned fieldLclNum = varDsc->lvFieldLclStart + i; + LclVarDsc* fieldVarDsc = compiler->lvaGetDesc(fieldLclNum); + var_types type = fieldVarDsc->TypeGet(); + if (varReg != REG_NA) + { + hasRegs = true; + if (varReg != reg) + { + inst_RV_RV(ins_Copy(type), varReg, reg, type); + } + fieldVarDsc->SetRegNum(varReg); + } + else + { + if (!lclNode->AsLclVar()->IsLastUse(i)) + { + GetEmitter()->emitIns_S_R(ins_Store(type), emitTypeSize(type), reg, fieldLclNum, 0); + } + fieldVarDsc->SetRegNum(REG_STK); + } + } + else + { + GetEmitter()->emitIns_S_R(ins_Store(type), emitTypeSize(type), reg, lclNum, offset); + offset += genTypeSize(type); + } + } + + // Update variable liveness. + if (isMultiRegVar) + { + if (hasRegs) + { + genProduceReg(lclNode); + } + else + { + genUpdateLife(lclNode); + } + } + else + { + genUpdateLife(lclNode); + varDsc->SetRegNum(REG_STK); + } +} + //------------------------------------------------------------------------ // genRegCopy: Produce code for a GT_COPY node. // @@ -11723,9 +11885,9 @@ void CodeGen::genStructReturn(GenTree* treeNode) // tree - the GT_COPY node // // Notes: -// This will copy the register(s) produced by this nodes source, to -// the register(s) allocated to this GT_COPY node. -// It has some special handling for these casess: +// This will copy the register produced by this node's source, to +// the register allocated to this GT_COPY node. +// It has some special handling for these cases: // - when the source and target registers are in different register files // (note that this is *not* a conversion). // - when the source is a lclVar whose home location is being moved to a new @@ -11751,7 +11913,10 @@ void CodeGen::genRegCopy(GenTree* treeNode) GenTreeCopyOrReload* copyNode = treeNode->AsCopyOrReload(); // GenTreeCopyOrReload only reports the highest index that has a valid register. - unsigned regCount = copyNode->GetRegCount(); + // However, we need to ensure that we consume all the registers of the child node, + // so we use its regCount. + unsigned regCount = + op1->IsMultiRegLclVar() ? op1->AsLclVar()->GetFieldCount(compiler) : op1->GetMultiRegCount(); assert(regCount <= MAX_MULTIREG_COUNT); // First set the source registers as busy if they haven't been spilled. @@ -11764,114 +11929,96 @@ void CodeGen::genRegCopy(GenTree* treeNode) busyRegs |= genRegMask(op1->GetRegByIndex(i)); } } - // First do any copies - we'll do the reloads after all the copies are complete. for (unsigned i = 0; i < regCount; ++i) { regNumber sourceReg = op1->GetRegByIndex(i); - regNumber targetReg = copyNode->GetRegNumByIdx(i); - // GenTreeCopyOrReload only reports the highest index that has a valid register. - // However there may be lower indices that have no valid register (i.e. the register - // on the source is still valid at the consumer). - if (targetReg != REG_NA) + // genRegCopy will consume the source register, perform any required reloads, + // and will return either the register copied to, or the original register if there's no copy. + regNumber targetReg = genRegCopy(treeNode, i); + if (targetReg != sourceReg) { - // We shouldn't specify a no-op move. regMaskTP targetRegMask = genRegMask(targetReg); - assert(sourceReg != targetReg); assert((busyRegs & targetRegMask) == 0); // Clear sourceReg from the busyRegs, and add targetReg. busyRegs &= ~genRegMask(sourceReg); - busyRegs |= genRegMask(targetReg); - var_types type; - if (op1->IsMultiRegLclVar()) - { - type = op1->AsLclVar()->GetFieldTypeByIndex(compiler, i); - } - else - { - type = op1->GetRegTypeByIndex(i); - } - inst_RV_RV(ins_Copy(type), targetReg, sourceReg, type); } + busyRegs |= genRegMask(targetReg); } - // Now we can consume op1, which will perform any necessary reloads. - genConsumeReg(op1); + return; } - else - { - var_types targetType = treeNode->TypeGet(); - regNumber targetReg = treeNode->GetRegNum(); - assert(targetReg != REG_NA); - assert(targetType != TYP_STRUCT); + var_types targetType = treeNode->TypeGet(); + regNumber targetReg = treeNode->GetRegNum(); + assert(targetReg != REG_NA); + assert(targetType != TYP_STRUCT); - // Check whether this node and the node from which we're copying the value have - // different register types. This can happen if (currently iff) we have a SIMD - // vector type that fits in an integer register, in which case it is passed as - // an argument, or returned from a call, in an integer register and must be - // copied if it's in an xmm register. + // Check whether this node and the node from which we're copying the value have + // different register types. This can happen if (currently iff) we have a SIMD + // vector type that fits in an integer register, in which case it is passed as + // an argument, or returned from a call, in an integer register and must be + // copied if it's in an xmm register. - bool srcFltReg = (varTypeIsFloating(op1) || varTypeIsSIMD(op1)); - bool tgtFltReg = (varTypeIsFloating(treeNode) || varTypeIsSIMD(treeNode)); - if (srcFltReg != tgtFltReg) + bool srcFltReg = (varTypeIsFloating(op1) || varTypeIsSIMD(op1)); + bool tgtFltReg = (varTypeIsFloating(treeNode) || varTypeIsSIMD(treeNode)); + if (srcFltReg != tgtFltReg) + { + instruction ins; + regNumber fpReg; + regNumber intReg; + if (tgtFltReg) { - instruction ins; - regNumber fpReg; - regNumber intReg; - if (tgtFltReg) - { - ins = ins_CopyIntToFloat(op1->TypeGet(), treeNode->TypeGet()); - fpReg = targetReg; - intReg = op1->GetRegNum(); - } - else - { - ins = ins_CopyFloatToInt(op1->TypeGet(), treeNode->TypeGet()); - intReg = targetReg; - fpReg = op1->GetRegNum(); - } - inst_RV_RV(ins, fpReg, intReg, targetType); + ins = ins_CopyIntToFloat(op1->TypeGet(), treeNode->TypeGet()); + fpReg = targetReg; + intReg = op1->GetRegNum(); } else { - inst_RV_RV(ins_Copy(targetType), targetReg, genConsumeReg(op1), targetType); + ins = ins_CopyFloatToInt(op1->TypeGet(), treeNode->TypeGet()); + intReg = targetReg; + fpReg = op1->GetRegNum(); } + inst_RV_RV(ins, fpReg, intReg, targetType); + } + else + { + inst_RV_RV(ins_Copy(targetType), targetReg, genConsumeReg(op1), targetType); + } - if (op1->IsLocal()) - { - // The lclVar will never be a def. - // If it is a last use, the lclVar will be killed by genConsumeReg(), as usual, and genProduceReg will - // appropriately set the gcInfo for the copied value. - // If not, there are two cases we need to handle: - // - If this is a TEMPORARY copy (indicated by the GTF_VAR_DEATH flag) the variable - // will remain live in its original register. - // genProduceReg() will appropriately set the gcInfo for the copied value, - // and genConsumeReg will reset it. - // - Otherwise, we need to update register info for the lclVar. + if (op1->IsLocal()) + { + // The lclVar will never be a def. + // If it is a last use, the lclVar will be killed by genConsumeReg(), as usual, and genProduceReg will + // appropriately set the gcInfo for the copied value. + // If not, there are two cases we need to handle: + // - If this is a TEMPORARY copy (indicated by the GTF_VAR_DEATH flag) the variable + // will remain live in its original register. + // genProduceReg() will appropriately set the gcInfo for the copied value, + // and genConsumeReg will reset it. + // - Otherwise, we need to update register info for the lclVar. - GenTreeLclVarCommon* lcl = op1->AsLclVarCommon(); - assert((lcl->gtFlags & GTF_VAR_DEF) == 0); + GenTreeLclVarCommon* lcl = op1->AsLclVarCommon(); + assert((lcl->gtFlags & GTF_VAR_DEF) == 0); - if ((lcl->gtFlags & GTF_VAR_DEATH) == 0 && (treeNode->gtFlags & GTF_VAR_DEATH) == 0) - { - LclVarDsc* varDsc = compiler->lvaGetDesc(lcl); + if ((lcl->gtFlags & GTF_VAR_DEATH) == 0 && (treeNode->gtFlags & GTF_VAR_DEATH) == 0) + { + LclVarDsc* varDsc = compiler->lvaGetDesc(lcl); - // If we didn't just spill it (in genConsumeReg, above), then update the register info - if (varDsc->GetRegNum() != REG_STK) - { - // The old location is dying - genUpdateRegLife(varDsc, /*isBorn*/ false, /*isDying*/ true DEBUGARG(op1)); + // If we didn't just spill it (in genConsumeReg, above), then update the register info + if (varDsc->GetRegNum() != REG_STK) + { + // The old location is dying + genUpdateRegLife(varDsc, /*isBorn*/ false, /*isDying*/ true DEBUGARG(op1)); - gcInfo.gcMarkRegSetNpt(genRegMask(op1->GetRegNum())); + gcInfo.gcMarkRegSetNpt(genRegMask(op1->GetRegNum())); - genUpdateVarReg(varDsc, treeNode); + genUpdateVarReg(varDsc, treeNode); #ifdef USING_VARIABLE_LIVE_RANGE - // Report the home change for this variable - varLiveKeeper->siUpdateVariableLiveRange(varDsc, lcl->GetLclNum()); + // Report the home change for this variable + varLiveKeeper->siUpdateVariableLiveRange(varDsc, lcl->GetLclNum()); #endif // USING_VARIABLE_LIVE_RANGE - // The new location is going live - genUpdateRegLife(varDsc, /*isBorn*/ true, /*isDying*/ false DEBUGARG(treeNode)); - } + // The new location is going live + genUpdateRegLife(varDsc, /*isBorn*/ true, /*isDying*/ false DEBUGARG(treeNode)); } } } @@ -11879,6 +12026,86 @@ void CodeGen::genRegCopy(GenTree* treeNode) genProduceReg(treeNode); } +//------------------------------------------------------------------------ +// genRegCopy: Produce code for a single register of a multireg copy node. +// +// Arguments: +// tree - The GT_COPY node +// multiRegIndex - The index of the register to be copied +// +// Notes: +// This will copy the corresponding register produced by this node's source, to +// the register allocated to the register specified by this GT_COPY node. +// A multireg copy doesn't support moving between register files, as the GT_COPY +// node does not retain separate types for each index. +// - when the source is a lclVar whose home location is being moved to a new +// register (rather than just being copied for temporary use). +// +// Return Value: +// Either the register copied to, or the original register if there's no copy. +// +regNumber CodeGen::genRegCopy(GenTree* treeNode, unsigned multiRegIndex) +{ + assert(treeNode->OperGet() == GT_COPY); + GenTree* op1 = treeNode->gtGetOp1(); + assert(op1->IsMultiRegNode()); + + GenTreeCopyOrReload* copyNode = treeNode->AsCopyOrReload(); + assert(copyNode->GetRegCount() <= MAX_MULTIREG_COUNT); + + // Consume op1's register, which will perform any necessary reloads. + genConsumeReg(op1, multiRegIndex); + + regNumber sourceReg = op1->GetRegByIndex(multiRegIndex); + regNumber targetReg = copyNode->GetRegNumByIdx(multiRegIndex); + // GenTreeCopyOrReload only reports the highest index that has a valid register. + // However there may be lower indices that have no valid register (i.e. the register + // on the source is still valid at the consumer). + if (targetReg != REG_NA) + { + // We shouldn't specify a no-op move. + regMaskTP targetRegMask = genRegMask(targetReg); + assert(sourceReg != targetReg); + var_types type; + if (op1->IsMultiRegLclVar()) + { + LclVarDsc* parentVarDsc = compiler->lvaGetDesc(op1->AsLclVar()->GetLclNum()); + unsigned fieldVarNum = parentVarDsc->lvFieldLclStart + multiRegIndex; + LclVarDsc* fieldVarDsc = compiler->lvaGetDesc(fieldVarNum); + type = fieldVarDsc->TypeGet(); + inst_RV_RV(ins_Copy(type), targetReg, sourceReg, type); + if (!op1->AsLclVar()->IsLastUse(multiRegIndex) && fieldVarDsc->GetRegNum() != REG_STK) + { + // The old location is dying + genUpdateRegLife(fieldVarDsc, /*isBorn*/ false, /*isDying*/ true DEBUGARG(op1)); + gcInfo.gcMarkRegSetNpt(genRegMask(sourceReg)); + genUpdateVarReg(fieldVarDsc, treeNode); + +#ifdef USING_VARIABLE_LIVE_RANGE + // Report the home change for this variable + varLiveKeeper->siUpdateVariableLiveRange(fieldVarDsc, fieldVarNum); +#endif // USING_VARIABLE_LIVE_RANGE + + // The new location is going live + genUpdateRegLife(fieldVarDsc, /*isBorn*/ true, /*isDying*/ false DEBUGARG(treeNode)); + } + } + else + { + type = op1->GetRegTypeByIndex(multiRegIndex); + inst_RV_RV(ins_Copy(type), targetReg, sourceReg, type); + // We never spill after a copy, so to produce the single register, we simply need to + // update the GC info for the defined register. + gcInfo.gcMarkRegPtrVal(targetReg, type); + } + return targetReg; + } + else + { + return sourceReg; + } +} + #if defined(DEBUG) && defined(TARGET_XARCH) //------------------------------------------------------------------------ diff --git a/src/coreclr/src/jit/codegeninterface.h b/src/coreclr/src/jit/codegeninterface.h index 2b484809d2841..3e588b94ba8a0 100644 --- a/src/coreclr/src/jit/codegeninterface.h +++ b/src/coreclr/src/jit/codegeninterface.h @@ -129,6 +129,7 @@ class CodeGenInterface // Liveness-related fields & methods public: void genUpdateRegLife(const LclVarDsc* varDsc, bool isBorn, bool isDying DEBUGARG(GenTree* tree)); + void genUpdateVarReg(LclVarDsc* varDsc, GenTree* tree, int regIndex); void genUpdateVarReg(LclVarDsc* varDsc, GenTree* tree); protected: diff --git a/src/coreclr/src/jit/codegenlinear.cpp b/src/coreclr/src/jit/codegenlinear.cpp index 5835feb0ffb46..e25485269bffe 100644 --- a/src/coreclr/src/jit/codegenlinear.cpp +++ b/src/coreclr/src/jit/codegenlinear.cpp @@ -885,6 +885,23 @@ void CodeGen::genSpillVar(GenTree* tree) #endif // USING_VARIABLE_LIVE_RANGE } +//------------------------------------------------------------------------ +// genUpdateVarReg: Update the current register location for a multi-reg lclVar +// +// Arguments: +// varDsc - the LclVarDsc for the lclVar +// tree - the lclVar node +// regIndex - the index of the register in the node +// +// inline +void CodeGenInterface::genUpdateVarReg(LclVarDsc* varDsc, GenTree* tree, int regIndex) +{ + // This should only be called for multireg lclVars. + assert(compiler->lvaEnregMultiRegVars); + assert(tree->IsMultiRegLclVar() || (tree->gtOper == GT_COPY)); + varDsc->SetRegNum(tree->GetRegByIndex(regIndex)); +} + //------------------------------------------------------------------------ // genUpdateVarReg: Update the current register location for a lclVar // @@ -895,7 +912,8 @@ void CodeGen::genSpillVar(GenTree* tree) // inline void CodeGenInterface::genUpdateVarReg(LclVarDsc* varDsc, GenTree* tree) { - assert(tree->OperIsScalarLocal() || (tree->gtOper == GT_COPY)); + // This should not be called for multireg lclVars. + assert((tree->OperIsScalarLocal() && !tree->IsMultiRegLclVar()) || (tree->gtOper == GT_COPY)); varDsc->SetRegNum(tree->GetRegNum()); } @@ -939,7 +957,7 @@ GenTree* sameRegAsDst(GenTree* tree, GenTree*& other /*out*/) } //------------------------------------------------------------------------ -// genUnspillLocal: Reload a register candidate local into a register. +// genUnspillLocal: Reload a register candidate local into a register, if needed. // // Arguments: // varNum - The variable number of the local to be reloaded (unspilled). @@ -1018,6 +1036,70 @@ void CodeGen::genUnspillLocal( gcInfo.gcMarkRegPtrVal(regNum, type); } +//------------------------------------------------------------------------ +// genUnspillRegIfNeeded: Reload a MultiReg source value into a register, if needed +// +// Arguments: +// tree - the MultiReg node of interest. +// multiRegIndex - the index of the value to reload, if needed. +// +// Notes: +// It must *not* be a GT_LCL_VAR (those are handled separately). +// In the normal case, the value will be reloaded into the register it +// was originally computed into. However, if that register is not available, +// the register allocator will have allocated a different register, and +// inserted a GT_RELOAD to indicate the register into which it should be +// reloaded. +// +void CodeGen::genUnspillRegIfNeeded(GenTree* tree, unsigned multiRegIndex) +{ + GenTree* unspillTree = tree; + assert(unspillTree->IsMultiRegNode()); + + if (tree->gtOper == GT_RELOAD) + { + unspillTree = tree->AsOp()->gtOp1; + } + + // In case of multi-reg node, GTF_SPILLED flag on it indicates that + // one or more of its result regs are spilled. Individual spill flags need to be + // queried to determine which specific result regs need to be unspilled. + if ((unspillTree->gtFlags & GTF_SPILLED) == 0) + { + return; + } + unsigned spillFlags = unspillTree->GetRegSpillFlagByIdx(multiRegIndex); + if ((spillFlags & GTF_SPILLED) == 0) + { + return; + } + + regNumber dstReg = tree->GetRegByIndex(multiRegIndex); + if (dstReg == REG_NA) + { + assert(tree->IsCopyOrReload()); + dstReg = unspillTree->GetRegByIndex(multiRegIndex); + } + if (tree->IsMultiRegLclVar()) + { + GenTreeLclVar* lclNode = tree->AsLclVar(); + unsigned fieldVarNum = compiler->lvaGetDesc(lclNode)->lvFieldLclStart + multiRegIndex; + bool reSpill = ((spillFlags & GTF_SPILL) != 0); + bool isLastUse = lclNode->IsLastUse(multiRegIndex); + genUnspillLocal(fieldVarNum, compiler->lvaGetDesc(fieldVarNum)->TypeGet(), lclNode, dstReg, reSpill, isLastUse); + } + else + { + var_types dstType = unspillTree->GetRegTypeByIndex(multiRegIndex); + regNumber unspillTreeReg = unspillTree->GetRegByIndex(multiRegIndex); + TempDsc* t = regSet.rsUnspillInPlace(unspillTree, unspillTreeReg, multiRegIndex); + emitAttr emitType = emitActualTypeSize(dstType); + GetEmitter()->emitIns_R_S(ins_Load(dstType), emitType, dstReg, t->tdTempNum(), 0); + regSet.tmpRlsTemp(t); + gcInfo.gcMarkRegPtrVal(dstReg, dstType); + } +} + //------------------------------------------------------------------------ // genUnspillRegIfNeeded: Reload the value into a register, if needed // @@ -1085,114 +1167,42 @@ void CodeGen::genUnspillRegIfNeeded(GenTree* tree) #endif bool reSpill = ((unspillTree->gtFlags & GTF_SPILL) != 0); bool isLastUse = lcl->IsLastUse(0); - genUnspillLocal(lcl->GetLclNum(), spillType, lcl, dstReg, reSpill, isLastUse); + genUnspillLocal(lcl->GetLclNum(), spillType, lcl->AsLclVar(), dstReg, reSpill, isLastUse); } - else if (unspillTree->IsMultiRegCall()) + else if (unspillTree->IsMultiRegLclVar()) { - GenTreeCall* call = unspillTree->AsCall(); - const ReturnTypeDesc* retTypeDesc = call->GetReturnTypeDesc(); - const unsigned regCount = retTypeDesc->GetReturnRegCount(); - GenTreeCopyOrReload* reloadTree = nullptr; - if (tree->OperGet() == GT_RELOAD) - { - reloadTree = tree->AsCopyOrReload(); - } + GenTreeLclVar* lclNode = unspillTree->AsLclVar(); + LclVarDsc* varDsc = compiler->lvaGetDesc(lclNode->GetLclNum()); + unsigned regCount = varDsc->lvFieldCnt; - // In case of multi-reg call node, GTF_SPILLED flag on it indicates that - // one or more of its result regs are spilled. Call node needs to be - // queried to know which specific result regs to be unspilled. for (unsigned i = 0; i < regCount; ++i) { - unsigned flags = call->GetRegSpillFlagByIdx(i); - if ((flags & GTF_SPILLED) != 0) + unsigned spillFlags = lclNode->GetRegSpillFlagByIdx(i); + if ((spillFlags & GTF_SPILLED) != 0) { - var_types dstType = retTypeDesc->GetReturnRegType(i); - regNumber unspillTreeReg = call->GetRegNumByIdx(i); - - if (reloadTree != nullptr) - { - dstReg = reloadTree->GetRegNumByIdx(i); - if (dstReg == REG_NA) - { - dstReg = unspillTreeReg; - } - } - else - { - dstReg = unspillTreeReg; - } - - TempDsc* t = regSet.rsUnspillInPlace(call, unspillTreeReg, i); - GetEmitter()->emitIns_R_S(ins_Load(dstType), emitActualTypeSize(dstType), dstReg, t->tdTempNum(), - 0); - regSet.tmpRlsTemp(t); - gcInfo.gcMarkRegPtrVal(dstReg, dstType); - } - } - - unspillTree->gtFlags &= ~GTF_SPILLED; - } -#if FEATURE_ARG_SPLIT - else if (unspillTree->OperIsPutArgSplit()) - { - GenTreePutArgSplit* splitArg = unspillTree->AsPutArgSplit(); - unsigned regCount = splitArg->gtNumRegs; - - // In case of split struct argument node, GTF_SPILLED flag on it indicates that - // one or more of its result regs are spilled. Call node needs to be - // queried to know which specific result regs to be unspilled. - for (unsigned i = 0; i < regCount; ++i) - { - unsigned flags = splitArg->GetRegSpillFlagByIdx(i); - if ((flags & GTF_SPILLED) != 0) - { - var_types dstType = splitArg->GetRegType(i); - regNumber dstReg = splitArg->GetRegNumByIdx(i); - - TempDsc* t = regSet.rsUnspillInPlace(splitArg, dstReg, i); - GetEmitter()->emitIns_R_S(ins_Load(dstType), emitActualTypeSize(dstType), dstReg, t->tdTempNum(), - 0); - regSet.tmpRlsTemp(t); - gcInfo.gcMarkRegPtrVal(dstReg, dstType); + regNumber reg = lclNode->GetRegNumByIdx(i); + unsigned fieldVarNum = varDsc->lvFieldLclStart + i; + bool reSpill = ((spillFlags & GTF_SPILL) != 0); + bool isLastUse = lclNode->IsLastUse(i); + genUnspillLocal(fieldVarNum, compiler->lvaGetDesc(fieldVarNum)->TypeGet(), lclNode, reg, reSpill, + isLastUse); } } - - unspillTree->gtFlags &= ~GTF_SPILLED; } -#ifdef TARGET_ARM - else if (unspillTree->OperIsMultiRegOp()) + else if (unspillTree->IsMultiRegNode()) { - GenTreeMultiRegOp* multiReg = unspillTree->AsMultiRegOp(); - unsigned regCount = multiReg->GetRegCount(); - - // In case of split struct argument node, GTF_SPILLED flag on it indicates that - // one or more of its result regs are spilled. Call node needs to be - // queried to know which specific result regs to be unspilled. + unsigned regCount = unspillTree->GetMultiRegCount(); for (unsigned i = 0; i < regCount; ++i) { - unsigned flags = multiReg->GetRegSpillFlagByIdx(i); - if ((flags & GTF_SPILLED) != 0) - { - var_types dstType = multiReg->GetRegType(i); - regNumber dstReg = multiReg->GetRegNumByIdx(i); - - TempDsc* t = regSet.rsUnspillInPlace(multiReg, dstReg, i); - GetEmitter()->emitIns_R_S(ins_Load(dstType), emitActualTypeSize(dstType), dstReg, t->tdTempNum(), - 0); - regSet.tmpRlsTemp(t); - gcInfo.gcMarkRegPtrVal(dstReg, dstType); - } + genUnspillRegIfNeeded(unspillTree, i); } - unspillTree->gtFlags &= ~GTF_SPILLED; } -#endif // TARGET_ARM -#endif // FEATURE_ARG_SPLIT else { - TempDsc* t = regSet.rsUnspillInPlace(unspillTree, unspillTree->GetRegNum()); - GetEmitter()->emitIns_R_S(ins_Load(unspillTree->gtType), emitActualTypeSize(unspillTree->TypeGet()), dstReg, - t->tdTempNum(), 0); + TempDsc* t = regSet.rsUnspillInPlace(unspillTree, unspillTree->GetRegNum()); + emitAttr emitType = emitActualTypeSize(unspillTree->TypeGet()); + GetEmitter()->emitIns_R_S(ins_Load(unspillTree->gtType), emitType, dstReg, t->tdTempNum(), 0); regSet.tmpRlsTemp(t); unspillTree->gtFlags &= ~GTF_SPILLED; @@ -1293,6 +1303,70 @@ void CodeGen::genCheckConsumeNode(GenTree* const node) } #endif // DEBUG +//-------------------------------------------------------------------- +// genConsumeReg: Do liveness update for a single register of a multireg child node +// that is being consumed by codegen. +// +// Arguments: +// tree - GenTree node +// multiRegIndex - The index of the register to be consumed +// +// Return Value: +// Returns the reg number for the given multiRegIndex. +// +regNumber CodeGen::genConsumeReg(GenTree* tree, unsigned multiRegIndex) +{ + regNumber reg = tree->GetRegByIndex(multiRegIndex); + if (tree->OperIs(GT_COPY)) + { + reg = genRegCopy(tree, multiRegIndex); + } + else if (reg == REG_NA) + { + assert(tree->OperIs(GT_RELOAD)); + reg = tree->gtGetOp1()->GetRegByIndex(multiRegIndex); + assert(reg != REG_NA); + } + genUnspillRegIfNeeded(tree, multiRegIndex); + + // UpdateLifeFieldVar() will return true if local var should be spilled. + if (tree->IsMultiRegLclVar() && treeLifeUpdater->UpdateLifeFieldVar(tree->AsLclVar(), multiRegIndex)) + { + GenTreeLclVar* lcl = tree->AsLclVar(); + genSpillLocal(lcl->GetLclNum(), lcl->GetFieldTypeByIndex(compiler, multiRegIndex), lcl, + lcl->GetRegByIndex(multiRegIndex)); + } + + if (tree->gtSkipReloadOrCopy()->OperIs(GT_LCL_VAR)) + { + GenTreeLclVar* lcl = tree->gtSkipReloadOrCopy()->AsLclVar(); + LclVarDsc* varDsc = compiler->lvaGetDesc(lcl); + assert(compiler->lvaEnregMultiRegVars && lcl->IsMultiReg()); + assert(varDsc->lvPromoted && (multiRegIndex < varDsc->lvFieldCnt)); + unsigned fieldVarNum = varDsc->lvFieldLclStart + multiRegIndex; + LclVarDsc* fldVarDsc = compiler->lvaGetDesc(fieldVarNum); + assert(fldVarDsc->lvLRACandidate); + bool isInReg = fldVarDsc->lvIsInReg() && reg != REG_NA; + bool isInMemory = !isInReg || fldVarDsc->lvLiveInOutOfHndlr; + bool isFieldDying = lcl->IsLastUse(multiRegIndex); + + if (fldVarDsc->GetRegNum() == REG_STK) + { + // We have loaded this into a register only temporarily + gcInfo.gcMarkRegSetNpt(reg); + } + else if (isFieldDying) + { + gcInfo.gcMarkRegSetNpt(genRegMask(fldVarDsc->GetRegNum())); + } + } + else + { + gcInfo.gcMarkRegSetNpt(tree->gtGetRegMask()); + } + return reg; +} + //-------------------------------------------------------------------- // genConsumeReg: Do liveness update for a subnode that is being // consumed by codegen. @@ -1336,8 +1410,6 @@ regNumber CodeGen::genConsumeReg(GenTree* tree) // genUpdateLife() will also spill local var if marked as GTF_SPILL by calling CodeGen::genSpillVar genUpdateLife(tree); - assert(tree->gtHasReg()); - // there are three cases where consuming a reg means clearing the bit in the live mask // 1. it was not produced by a local // 2. it was produced by a local that is going dead @@ -1345,7 +1417,9 @@ regNumber CodeGen::genConsumeReg(GenTree* tree) if (genIsRegCandidateLocal(tree)) { - GenTreeLclVarCommon* lcl = tree->AsLclVarCommon(); + assert(tree->gtHasReg()); + + GenTreeLclVarCommon* lcl = tree->AsLclVar(); LclVarDsc* varDsc = &compiler->lvaTable[lcl->GetLclNum()]; assert(varDsc->lvLRACandidate); @@ -1359,6 +1433,40 @@ regNumber CodeGen::genConsumeReg(GenTree* tree) gcInfo.gcMarkRegSetNpt(genRegMask(varDsc->GetRegNum())); } } + else if (tree->gtSkipReloadOrCopy()->IsMultiRegLclVar()) + { + assert(compiler->lvaEnregMultiRegVars); + GenTreeLclVar* lcl = tree->gtSkipReloadOrCopy()->AsLclVar(); + LclVarDsc* varDsc = compiler->lvaGetDesc(lcl); + unsigned firstFieldVarNum = varDsc->lvFieldLclStart; + for (unsigned i = 0; i < varDsc->lvFieldCnt; ++i) + { + LclVarDsc* fldVarDsc = &(compiler->lvaTable[firstFieldVarNum + i]); + assert(fldVarDsc->lvLRACandidate); + regNumber reg; + if (tree->OperIs(GT_COPY, GT_RELOAD) && (tree->AsCopyOrReload()->GetRegByIndex(i) != REG_NA)) + { + reg = tree->AsCopyOrReload()->GetRegByIndex(i); + } + else + { + reg = lcl->AsLclVar()->GetRegNumByIdx(i); + } + bool isInReg = fldVarDsc->lvIsInReg() && reg != REG_NA; + bool isInMemory = !isInReg || fldVarDsc->lvLiveInOutOfHndlr; + bool isFieldDying = lcl->IsLastUse(i); + + if (fldVarDsc->GetRegNum() == REG_STK) + { + // We have loaded this into a register only temporarily + gcInfo.gcMarkRegSetNpt(reg); + } + else if (isFieldDying) + { + gcInfo.gcMarkRegSetNpt(genRegMask(fldVarDsc->GetRegNum())); + } + } + } else { gcInfo.gcMarkRegSetNpt(tree->gtGetRegMask()); @@ -1923,6 +2031,24 @@ void CodeGen::genProduceReg(GenTree* tree) unsigned varNum = tree->AsLclVarCommon()->GetLclNum(); genSpillLocal(varNum, tree->TypeGet(), tree->AsLclVar(), tree->GetRegNum()); } + else if (tree->IsMultiRegLclVar()) + { + assert(compiler->lvaEnregMultiRegVars); + GenTreeLclVar* lclNode = tree->AsLclVar(); + LclVarDsc* varDsc = compiler->lvaGetDesc(lclNode->GetLclNum()); + unsigned regCount = lclNode->GetFieldCount(compiler); + + for (unsigned i = 0; i < regCount; ++i) + { + unsigned flags = lclNode->GetRegSpillFlagByIdx(i); + if ((flags & GTF_SPILL) != 0) + { + regNumber reg = lclNode->GetRegNumByIdx(i); + unsigned fieldVarNum = varDsc->lvFieldLclStart + i; + genSpillLocal(fieldVarNum, compiler->lvaGetDesc(fieldVarNum)->TypeGet(), lclNode, reg); + } + } + } else { // In case of multi-reg call node, spill flag on call node @@ -2049,6 +2175,25 @@ void CodeGen::genProduceReg(GenTree* tree) } } } + else if (tree->IsMultiRegLclVar()) + { + assert(compiler->lvaEnregMultiRegVars); + GenTreeLclVar* lclNode = tree->AsLclVar(); + LclVarDsc* varDsc = compiler->lvaGetDesc(lclNode->GetLclNum()); + unsigned regCount = varDsc->lvFieldCnt; + for (unsigned i = 0; i < regCount; i++) + { + if (!lclNode->IsLastUse(i)) + { + regNumber reg = lclNode->GetRegByIndex(i); + if (reg != REG_NA) + { + var_types type = compiler->lvaGetDesc(varDsc->lvFieldLclStart + i)->TypeGet(); + gcInfo.gcMarkRegPtrVal(reg, type); + } + } + } + } else { gcInfo.gcMarkRegPtrVal(tree->GetRegNum(), tree->TypeGet()); diff --git a/src/coreclr/src/jit/codegenxarch.cpp b/src/coreclr/src/jit/codegenxarch.cpp index 4d8d2fbf9e217..d392137f89a57 100644 --- a/src/coreclr/src/jit/codegenxarch.cpp +++ b/src/coreclr/src/jit/codegenxarch.cpp @@ -1849,144 +1849,100 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) } } +#ifdef FEATURE_SIMD //---------------------------------------------------------------------------------- -// genMultiRegStoreToLocal: store multi-reg return value of a call node to a local +// genMultiRegStoreToSIMDLocal: store multi-reg value to a single-reg SIMD local // // Arguments: -// treeNode - Gentree of GT_STORE_LCL_VAR +// lclNode - GentreeLclVar of GT_STORE_LCL_VAR // // Return Value: // None // -// Assumptions: -// The child of store is a multi-reg node. -// -void CodeGen::genMultiRegStoreToLocal(GenTree* treeNode) +void CodeGen::genMultiRegStoreToSIMDLocal(GenTreeLclVar* lclNode) { - assert(treeNode->OperGet() == GT_STORE_LCL_VAR); - assert(varTypeIsStruct(treeNode) || varTypeIsMultiReg(treeNode)); - GenTree* op1 = treeNode->gtGetOp1(); - GenTree* actualOp1 = op1->gtSkipReloadOrCopy(); +#ifdef UNIX_AMD64_ABI + regNumber dst = lclNode->GetRegNum(); + GenTree* op1 = lclNode->gtGetOp1(); + GenTree* actualOp1 = op1->gtSkipReloadOrCopy(); + unsigned regCount = + actualOp1->IsMultiRegLclVar() ? actualOp1->AsLclVar()->GetFieldCount(compiler) : actualOp1->GetMultiRegCount(); assert(op1->IsMultiRegNode()); - unsigned regCount = actualOp1->GetMultiRegCount(); - - // Assumption: The current implementation requires that a multi-reg - // var in 'var = call' is flagged as lvIsMultiRegRet to prevent it from - // being struct promoted. - - unsigned lclNum = treeNode->AsLclVarCommon()->GetLclNum(); - LclVarDsc* varDsc = compiler->lvaGetDesc(lclNum); - if (op1->OperIs(GT_CALL)) - { - assert(regCount == MAX_RET_REG_COUNT); - noway_assert(varDsc->lvIsMultiRegRet); - } - genConsumeRegs(op1); -#ifdef UNIX_AMD64_ABI - // Structs of size >=9 and <=16 are returned in two return registers on x64 Unix. - - // Handle the case of a SIMD type returned in 2 registers. - if (varTypeIsSIMD(treeNode) && (treeNode->GetRegNum() != REG_NA)) - { - // Right now the only enregistrable structs supported are SIMD types. - // They are only returned in 1 or 2 registers - the 1 register case is - // handled as a regular STORE_LCL_VAR. - // This case is always a call (AsCall() will assert if it is not). - GenTreeCall* call = actualOp1->AsCall(); - const ReturnTypeDesc* retTypeDesc = call->GetReturnTypeDesc(); - assert(retTypeDesc->GetReturnRegCount() == MAX_RET_REG_COUNT); - - assert(regCount == 2); - assert(varTypeIsFloating(retTypeDesc->GetReturnRegType(0))); - assert(varTypeIsFloating(retTypeDesc->GetReturnRegType(1))); - - // This is a case where the two 8-bytes that comprise the operand are in - // two different xmm registers and need to be assembled into a single - // xmm register. - regNumber targetReg = treeNode->GetRegNum(); - regNumber reg0 = call->GetRegNumByIdx(0); - regNumber reg1 = call->GetRegNumByIdx(1); - - if (op1->IsCopyOrReload()) - { - // GT_COPY/GT_RELOAD will have valid reg for those positions - // that need to be copied or reloaded. - regNumber reloadReg = op1->AsCopyOrReload()->GetRegNumByIdx(0); - if (reloadReg != REG_NA) - { - reg0 = reloadReg; - } + // Right now the only enregistrable structs supported are SIMD types. + // They are only returned in 1 or 2 registers - the 1 register case is + // handled as a regular STORE_LCL_VAR. + // This case is always a call (AsCall() will assert if it is not). + GenTreeCall* call = actualOp1->AsCall(); + const ReturnTypeDesc* retTypeDesc = call->GetReturnTypeDesc(); + assert(retTypeDesc->GetReturnRegCount() == MAX_RET_REG_COUNT); - reloadReg = op1->AsCopyOrReload()->GetRegNumByIdx(1); - if (reloadReg != REG_NA) - { - reg1 = reloadReg; - } - } + assert(regCount == 2); + assert(varTypeIsFloating(retTypeDesc->GetReturnRegType(0))); + assert(varTypeIsFloating(retTypeDesc->GetReturnRegType(1))); - if (targetReg != reg0 && targetReg != reg1) - { - // targetReg = reg0; - // targetReg[127:64] = reg1[127:64] - inst_RV_RV(ins_Copy(TYP_DOUBLE), targetReg, reg0, TYP_DOUBLE); - inst_RV_RV_IV(INS_shufpd, EA_16BYTE, targetReg, reg1, 0x00); - } - else if (targetReg == reg0) + // This is a case where the two 8-bytes that comprise the operand are in + // two different xmm registers and need to be assembled into a single + // xmm register. + regNumber targetReg = lclNode->GetRegNum(); + regNumber reg0 = call->GetRegNumByIdx(0); + regNumber reg1 = call->GetRegNumByIdx(1); + + if (op1->IsCopyOrReload()) + { + // GT_COPY/GT_RELOAD will have valid reg for those positions + // that need to be copied or reloaded. + regNumber reloadReg = op1->AsCopyOrReload()->GetRegNumByIdx(0); + if (reloadReg != REG_NA) { - // (elided) targetReg = reg0 - // targetReg[127:64] = reg1[127:64] - inst_RV_RV_IV(INS_shufpd, EA_16BYTE, targetReg, reg1, 0x00); + reg0 = reloadReg; } - else + + reloadReg = op1->AsCopyOrReload()->GetRegNumByIdx(1); + if (reloadReg != REG_NA) { - assert(targetReg == reg1); - // We need two shuffles to achieve this - // First: - // targetReg[63:0] = targetReg[63:0] - // targetReg[127:64] = reg0[63:0] - // - // Second: - // targetReg[63:0] = targetReg[127:64] - // targetReg[127:64] = targetReg[63:0] - // - // Essentially copy low 8-bytes from reg0 to high 8-bytes of targetReg - // and next swap low and high 8-bytes of targetReg to have them - // rearranged in the right order. - inst_RV_RV_IV(INS_shufpd, EA_16BYTE, targetReg, reg0, 0x00); - inst_RV_RV_IV(INS_shufpd, EA_16BYTE, targetReg, targetReg, 0x01); + reg1 = reloadReg; } } + + if (targetReg != reg0 && targetReg != reg1) + { + // targetReg = reg0; + // targetReg[127:64] = reg1[127:64] + inst_RV_RV(ins_Copy(TYP_DOUBLE), targetReg, reg0, TYP_DOUBLE); + inst_RV_RV_IV(INS_shufpd, EA_16BYTE, targetReg, reg1, 0x00); + } + else if (targetReg == reg0) + { + // (elided) targetReg = reg0 + // targetReg[127:64] = reg1[127:64] + inst_RV_RV_IV(INS_shufpd, EA_16BYTE, targetReg, reg1, 0x00); + } else -#endif // UNIX_AMD64_ABI { - // This may be: - // - a call returning multiple registers - // - a HW intrinsic producing two registers to be stored into a TYP_STRUCT + assert(targetReg == reg1); + // We need two shuffles to achieve this + // First: + // targetReg[63:0] = targetReg[63:0] + // targetReg[127:64] = reg0[63:0] // - int offset = 0; - for (unsigned i = 0; i < regCount; ++i) - { - var_types type = actualOp1->GetRegTypeByIndex(i); - regNumber reg = op1->GetRegByIndex(i); - if (reg == REG_NA) - { - // GT_COPY/GT_RELOAD will have valid reg only for those positions - // that need to be copied or reloaded. - assert(op1->IsCopyOrReload()); - reg = actualOp1->GetRegByIndex(i); - } - - assert(reg != REG_NA); - GetEmitter()->emitIns_S_R(ins_Store(type), emitTypeSize(type), reg, lclNum, offset); - offset += genTypeSize(type); - } - // Update variable liveness. - genUpdateLife(treeNode); - varDsc->SetRegNum(REG_STK); + // Second: + // targetReg[63:0] = targetReg[127:64] + // targetReg[127:64] = targetReg[63:0] + // + // Essentially copy low 8-bytes from reg0 to high 8-bytes of targetReg + // and next swap low and high 8-bytes of targetReg to have them + // rearranged in the right order. + inst_RV_RV_IV(INS_shufpd, EA_16BYTE, targetReg, reg0, 0x00); + inst_RV_RV_IV(INS_shufpd, EA_16BYTE, targetReg, targetReg, 0x01); } + genProduceReg(lclNode); +#else // !UNIX_AMD64_ABI + assert(!"Multireg store to SIMD reg not supported on X64 Windows"); +#endif // !UNIX_AMD64_ABI } +#endif // FEATURE_SIMD //------------------------------------------------------------------------ // genAllocLclFrame: Probe the stack and allocate the local stack frame - subtract from SP. @@ -4372,7 +4328,7 @@ void CodeGen::genCodeForLclVar(GenTreeLclVar* tree) // If this is a register candidate that has been spilled, genConsumeReg() will // reload it at the point of use. Otherwise, if it's not in a register, we load it here. - if (!isRegCandidate && !(tree->gtFlags & GTF_SPILLED)) + if (!isRegCandidate && !tree->IsMultiReg() && !(tree->gtFlags & GTF_SPILLED)) { #if defined(FEATURE_SIMD) && defined(TARGET_X86) // Loading of TYP_SIMD12 (i.e. Vector3) variable @@ -4428,29 +4384,28 @@ void CodeGen::genCodeForStoreLclFld(GenTreeLclFld* tree) // genCodeForStoreLclVar: Produce code for a GT_STORE_LCL_VAR node. // // Arguments: -// tree - the GT_STORE_LCL_VAR node +// lclNode - the GT_STORE_LCL_VAR node // -void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* tree) +void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* lclNode) { - assert(tree->OperIs(GT_STORE_LCL_VAR)); + assert(lclNode->OperIs(GT_STORE_LCL_VAR)); - regNumber targetReg = tree->GetRegNum(); + regNumber targetReg = lclNode->GetRegNum(); emitter* emit = GetEmitter(); - GenTree* op1 = tree->gtGetOp1(); + GenTree* op1 = lclNode->gtGetOp1(); - // var = call, where call returns a multi-reg return value - // case is handled separately. + // Stores from a multi-reg source are handled separately. if (op1->gtSkipReloadOrCopy()->IsMultiRegNode()) { - genMultiRegStoreToLocal(tree); + genMultiRegStoreToLocal(lclNode); } else { - unsigned lclNum = tree->GetLclNum(); + unsigned lclNum = lclNode->GetLclNum(); LclVarDsc* varDsc = compiler->lvaGetDesc(lclNum); - var_types targetType = varDsc->GetRegisterType(tree); + var_types targetType = varDsc->GetRegisterType(lclNode); #ifdef DEBUG var_types op1Type = op1->TypeGet(); @@ -4469,7 +4424,7 @@ void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* tree) #if !defined(TARGET_64BIT) if (targetType == TYP_LONG) { - genStoreLongLclVar(tree); + genStoreLongLclVar(lclNode); return; } #endif // !defined(TARGET_64BIT) @@ -4478,7 +4433,7 @@ void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* tree) // storing of TYP_SIMD12 (i.e. Vector3) field if (targetType == TYP_SIMD12) { - genStoreLclTypeSIMD12(tree); + genStoreLclTypeSIMD12(lclNode); return; } #endif // FEATURE_SIMD @@ -4494,7 +4449,7 @@ void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* tree) { emit->emitIns_S_R(ins_Store(srcType, compiler->isSIMDTypeLocalAligned(lclNum)), emitTypeSize(targetType), bitCastSrc->GetRegNum(), lclNum, 0); - genUpdateLife(tree); + genUpdateLife(lclNode); varDsc->SetRegNum(REG_STK); } else @@ -4506,7 +4461,7 @@ void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* tree) { // stack store emit->emitInsStoreLcl(ins_Store(targetType, compiler->isSIMDTypeLocalAligned(lclNum)), - emitTypeSize(targetType), tree); + emitTypeSize(targetType), lclNode); varDsc->SetRegNum(REG_STK); } else @@ -4538,14 +4493,13 @@ void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* tree) else if (op1->GetRegNum() != targetReg) { assert(op1->GetRegNum() != REG_NA); - emit->emitInsBinary(ins_Move_Extend(targetType, true), emitTypeSize(tree), tree, op1); + emit->emitInsBinary(ins_Move_Extend(targetType, true), emitTypeSize(lclNode), lclNode, op1); } } - } - - if (targetReg != REG_NA) - { - genProduceReg(tree); + if (targetReg != REG_NA) + { + genProduceReg(lclNode); + } } } diff --git a/src/coreclr/src/jit/compiler.cpp b/src/coreclr/src/jit/compiler.cpp index 0a9003f51268c..f94b5fba1f998 100644 --- a/src/coreclr/src/jit/compiler.cpp +++ b/src/coreclr/src/jit/compiler.cpp @@ -4460,6 +4460,35 @@ void Compiler::compCompile(void** methodCodePtr, ULONG* methodCodeSize, JitFlags printf(""); // flush } } + if (lvaEnregMultiRegVars) + { + unsigned methHash = info.compMethodHash(); + char* lostr = getenv("JitMultiRegHashLo"); + unsigned methHashLo = 0; + bool dump = false; + if (lostr != nullptr) + { + sscanf_s(lostr, "%x", &methHashLo); + dump = true; + } + char* histr = getenv("JitMultiRegHashHi"); + unsigned methHashHi = UINT32_MAX; + if (histr != nullptr) + { + sscanf_s(histr, "%x", &methHashHi); + dump = true; + } + if (methHash < methHashLo || methHash > methHashHi) + { + lvaEnregMultiRegVars = false; + } + else if (dump) + { + printf("Enregistering MultiReg Vars for method %s, hash = 0x%x.\n", info.compFullName, + info.compMethodHash()); + printf(""); // flush + } + } #endif // Compute bbNum, bbRefs and bbPreds diff --git a/src/coreclr/src/jit/compiler.h b/src/coreclr/src/jit/compiler.h index 0aadb65be3402..852b57f8359b8 100644 --- a/src/coreclr/src/jit/compiler.h +++ b/src/coreclr/src/jit/compiler.h @@ -3108,6 +3108,7 @@ class Compiler bool lvaVarDoNotEnregister(unsigned varNum); bool lvaEnregEHVars; + bool lvaEnregMultiRegVars; #ifdef DEBUG // Reasons why we can't enregister. Some of these correspond to debug properties of local vars. @@ -4600,7 +4601,7 @@ class Compiler VARSET_VALARG_TP keepAliveVars, LclVarDsc& varDsc, GenTreeLclVarCommon* node); - void fgComputeLifeUntrackedLocal(VARSET_TP& life, + bool fgComputeLifeUntrackedLocal(VARSET_TP& life, VARSET_VALARG_TP keepAliveVars, LclVarDsc& varDsc, GenTreeLclVarCommon* lclVarNode); diff --git a/src/coreclr/src/jit/gentree.cpp b/src/coreclr/src/jit/gentree.cpp index 2b7567f6d53ba..9c7b6b5b73440 100644 --- a/src/coreclr/src/jit/gentree.cpp +++ b/src/coreclr/src/jit/gentree.cpp @@ -679,7 +679,7 @@ bool GenTree::gtHasReg() const // This does not look at the actual register assignments, if any, and so // is valid after Lowering. // -int GenTree::GetRegisterDstCount() const +int GenTree::GetRegisterDstCount(Compiler* compiler) const { assert(!isContained()); if (!IsMultiRegNode()) @@ -692,7 +692,7 @@ int GenTree::GetRegisterDstCount() const } else if (IsCopyOrReload()) { - return gtGetOp1()->GetRegisterDstCount(); + return gtGetOp1()->GetRegisterDstCount(compiler); } #if FEATURE_ARG_SPLIT else if (OperIsPutArgSplit()) @@ -723,6 +723,10 @@ int GenTree::GetRegisterDstCount() const return 2; } #endif + if (OperIsScalarLocal()) + { + return AsLclVar()->GetFieldCount(compiler); + } assert(!"Unexpected multi-reg node"); return 0; } @@ -5569,7 +5573,7 @@ bool GenTree::OperMayThrow(Compiler* comp) // Notes: // This must be a multireg lclVar. // -unsigned int GenTreeLclVar::GetFieldCount(Compiler* compiler) +unsigned int GenTreeLclVar::GetFieldCount(Compiler* compiler) const { assert(IsMultiReg()); LclVarDsc* varDsc = compiler->lvaGetDesc(GetLclNum()); @@ -10208,10 +10212,12 @@ void Compiler::gtDispRegVal(GenTree* tree) } #if FEATURE_MULTIREG_RET - if (tree->IsMultiRegCall()) + if (tree->OperIs(GT_CALL)) { // 0th reg is GetRegNum(), which is already printed above. // Print the remaining regs of a multi-reg call node. + // Note that, prior to the initialization of the ReturnTypeDesc we won't print + // any additional registers. const GenTreeCall* call = tree->AsCall(); const unsigned regCount = call->GetReturnTypeDesc()->TryGetReturnRegCount(); for (unsigned i = 1; i < regCount; ++i) @@ -10226,14 +10232,11 @@ void Compiler::gtDispRegVal(GenTree* tree) unsigned regCount = 0; if (op1->OperIs(GT_CALL)) { - if (op1->IsMultiRegCall()) + regCount = op1->AsCall()->GetReturnTypeDesc()->TryGetReturnRegCount(); + // If it hasn't yet been initialized, we'd still like to see the registers printed. + if (regCount == 0) { - regCount = op1->AsCall()->GetReturnTypeDesc()->TryGetReturnRegCount(); - // If it hasn't yet been initialized, we'd still like to see the registers printed. - if (regCount == 0) - { - regCount = MAX_RET_REG_COUNT; - } + regCount = MAX_RET_REG_COUNT; } } else if (op1->IsMultiRegLclVar()) @@ -10249,7 +10252,7 @@ void Compiler::gtDispRegVal(GenTree* tree) for (unsigned i = 1; i < regCount; i++) { regNumber reg = tree->AsCopyOrReload()->GetRegNumByIdx(i); - printf(",%s", (reg == REG_NA) ? ",NA" : compRegVarName(reg)); + printf(",%s", (reg == REG_NA) ? "NA" : compRegVarName(reg)); } } #endif @@ -10846,8 +10849,8 @@ void Compiler::gtDispLeaf(GenTree* tree, IndentStack* indentStack) fieldVarDsc->PrintVarReg(); } - if (fieldVarDsc->lvTracked && fgLocalVarLivenessDone && // Includes local variable liveness - ((tree->gtFlags & GTF_VAR_DEATH) != 0)) + if (fieldVarDsc->lvTracked && fgLocalVarLivenessDone && tree->IsMultiRegLclVar() && + tree->AsLclVar()->IsLastUse(i - varDsc->lvFieldLclStart)) { printf(" (last use)"); } diff --git a/src/coreclr/src/jit/gentree.h b/src/coreclr/src/jit/gentree.h index 04617f5641bab..99c075ccfeaab 100644 --- a/src/coreclr/src/jit/gentree.h +++ b/src/coreclr/src/jit/gentree.h @@ -621,7 +621,7 @@ struct GenTree void CopyReg(GenTree* from); bool gtHasReg() const; - int GetRegisterDstCount() const; + int GetRegisterDstCount(Compiler* compiler) const; regMaskTP gtGetRegMask() const; @@ -799,7 +799,6 @@ struct GenTree #define GTF_VAR_ARR_INDEX 0x00000020 // The variable is part of (the index portion of) an array index expression. // Shares a value with GTF_REVERSE_OPS, which is meaningless for local var. - // For additional flags for GT_CALL node see GTF_CALL_M_* #define GTF_CALL_UNMANAGED 0x80000000 // GT_CALL -- direct call to unmanaged code @@ -3305,7 +3304,7 @@ struct GenTreeLclVar : public GenTreeLclVarCommon gtSpillFlags = SetMultiRegSpillFlagsByIdx(gtSpillFlags, flags, idx); } - unsigned int GetFieldCount(Compiler* compiler); + unsigned int GetFieldCount(Compiler* compiler) const; var_types GetFieldTypeByIndex(Compiler* compiler, unsigned idx); //------------------------------------------------------------------- diff --git a/src/coreclr/src/jit/importer.cpp b/src/coreclr/src/jit/importer.cpp index 9a279a3023209..1182d916cd0d6 100644 --- a/src/coreclr/src/jit/importer.cpp +++ b/src/coreclr/src/jit/importer.cpp @@ -1442,6 +1442,19 @@ GenTree* Compiler::impAssignStructPtr(GenTree* destAddr, { dest->gtType = asgType; } + if (dest->OperIs(GT_LCL_VAR) && + (src->IsMultiRegNode() || + (src->OperIs(GT_RET_EXPR) && src->AsRetExpr()->gtInlineCandidate->AsCall()->HasMultiRegRetVal()))) + { + if (lvaEnregMultiRegVars && varTypeIsStruct(dest)) + { + dest->AsLclVar()->SetMultiReg(); + } + if (src->OperIs(GT_CALL)) + { + lvaGetDesc(dest->AsLclVar())->lvIsMultiRegRet = true; + } + } dest->gtFlags |= destFlags; destFlags = dest->gtFlags; diff --git a/src/coreclr/src/jit/jitconfigvalues.h b/src/coreclr/src/jit/jitconfigvalues.h index b1ecf953479bd..92ad41c611886 100644 --- a/src/coreclr/src/jit/jitconfigvalues.h +++ b/src/coreclr/src/jit/jitconfigvalues.h @@ -246,6 +246,8 @@ CONFIG_INTEGER(EnableAVX, W("EnableAVX"), 0) CONFIG_INTEGER(EnableEHWriteThru, W("EnableEHWriteThru"), 0) // Enable the register allocator to support EH-write thru: // partial enregistration of vars exposed on EH boundaries +CONFIG_INTEGER(EnableMultiRegLocals, W("EnableMultiRegLocals"), 1) // Enable the enregistration of locals that are + // defined or used in a multireg context. // clang-format off diff --git a/src/coreclr/src/jit/lclvars.cpp b/src/coreclr/src/jit/lclvars.cpp index ecc99055cce93..b2c95af98915b 100644 --- a/src/coreclr/src/jit/lclvars.cpp +++ b/src/coreclr/src/jit/lclvars.cpp @@ -87,7 +87,8 @@ void Compiler::lvaInit() structPromotionHelper = new (this, CMK_Generic) StructPromotionHelper(this); - lvaEnregEHVars = (((opts.compFlags & CLFLG_REGVAR) != 0) && JitConfig.EnableEHWriteThru()); + lvaEnregEHVars = (((opts.compFlags & CLFLG_REGVAR) != 0) && JitConfig.EnableEHWriteThru()); + lvaEnregMultiRegVars = (((opts.compFlags & CLFLG_REGVAR) != 0) && JitConfig.EnableMultiRegLocals()); } /*****************************************************************************/ @@ -1844,17 +1845,9 @@ bool Compiler::StructPromotionHelper::CanPromoteStructVar(unsigned lclNum) return false; } -#if !FEATURE_MULTIREG_STRUCT_PROMOTE - if (varDsc->lvIsMultiRegArg) + if (!compiler->lvaEnregMultiRegVars && varDsc->lvIsMultiRegArgOrRet()) { - JITDUMP(" struct promotion of V%02u is disabled because lvIsMultiRegArg\n", lclNum); - return false; - } -#endif - - if (varDsc->lvIsMultiRegRet) - { - JITDUMP(" struct promotion of V%02u is disabled because lvIsMultiRegRet\n", lclNum); + JITDUMP(" struct promotion of V%02u is disabled because lvIsMultiRegArgOrRet()\n", lclNum); return false; } @@ -1867,7 +1860,26 @@ bool Compiler::StructPromotionHelper::CanPromoteStructVar(unsigned lclNum) CORINFO_CLASS_HANDLE typeHnd = varDsc->lvVerTypeInfo.GetClassHandle(); assert(typeHnd != nullptr); - return CanPromoteStructType(typeHnd); + + bool canPromote = CanPromoteStructType(typeHnd); + if (canPromote && varDsc->lvIsMultiRegArgOrRet()) + { + if (structPromotionInfo.fieldCnt > MAX_MULTIREG_COUNT) + { + canPromote = false; + } +#ifdef UNIX_AMD64_ABI + else + { + SortStructFields(); + if ((structPromotionInfo.fieldCnt == 2) && (structPromotionInfo.fields[1].fldOffset != TARGET_POINTER_SIZE)) + { + canPromote = false; + } + } +#endif // UNIX_AMD64_ABI + } + return canPromote; } //-------------------------------------------------------------------------------------------- @@ -1915,6 +1927,11 @@ bool Compiler::StructPromotionHelper::ShouldPromoteStructVar(unsigned lclNum) structPromotionInfo.fieldCnt, varDsc->lvFieldAccessed); shouldPromote = false; } + else if (varDsc->lvIsMultiRegRet && structPromotionInfo.containsHoles && structPromotionInfo.customLayout) + { + JITDUMP("Not promoting multi-reg returned struct local V%02u with holes.\n", lclNum); + shouldPromote = false; + } #if defined(TARGET_AMD64) || defined(TARGET_ARM64) || defined(TARGET_ARM) // TODO-PERF - Only do this when the LclVar is used in an argument context // TODO-ARM64 - HFA support should also eliminate the need for this. @@ -2292,12 +2309,13 @@ void Compiler::lvaPromoteLongVars() for (unsigned lclNum = 0; lclNum < startLvaCount; lclNum++) { LclVarDsc* varDsc = &lvaTable[lclNum]; - if (!varTypeIsLong(varDsc) || varDsc->lvDoNotEnregister || varDsc->lvIsMultiRegArgOrRet() || - (varDsc->lvRefCnt() == 0) || varDsc->lvIsStructField || (fgNoStructPromotion && varDsc->lvIsParam)) + if (!varTypeIsLong(varDsc) || varDsc->lvDoNotEnregister || (varDsc->lvRefCnt() == 0) || + varDsc->lvIsStructField || (fgNoStructPromotion && varDsc->lvIsParam)) { continue; } + assert(!varDsc->lvIsMultiRegArgOrRet()); varDsc->lvFieldCnt = 2; varDsc->lvFieldLclStart = lvaCount; varDsc->lvPromoted = true; diff --git a/src/coreclr/src/jit/liveness.cpp b/src/coreclr/src/jit/liveness.cpp index 96d623c221d5d..dc24bdffc2bab 100644 --- a/src/coreclr/src/jit/liveness.cpp +++ b/src/coreclr/src/jit/liveness.cpp @@ -1623,7 +1623,11 @@ bool Compiler::fgComputeLifeTrackedLocalDef(VARSET_TP& life, // keepAliveVars - The current set of variables to keep alive regardless of their actual lifetime. // varDsc - The LclVar descriptor for the variable being used or defined. // lclVarNode - The node that corresponds to the local var def or use. -void Compiler::fgComputeLifeUntrackedLocal(VARSET_TP& life, +// +// Returns: +// `true` if the node is a dead store (i.e. all fields are dead); `false` otherwise. +// +bool Compiler::fgComputeLifeUntrackedLocal(VARSET_TP& life, VARSET_VALARG_TP keepAliveVars, LclVarDsc& varDsc, GenTreeLclVarCommon* lclVarNode) @@ -1632,39 +1636,66 @@ void Compiler::fgComputeLifeUntrackedLocal(VARSET_TP& life, if (!varTypeIsStruct(varDsc.lvType) || (lvaGetPromotionType(&varDsc) == PROMOTION_TYPE_NONE)) { - return; + return false; } - VARSET_TP varBit(VarSetOps::MakeEmpty(this)); + VARSET_TP fieldSet(VarSetOps::MakeEmpty(this)); + bool fieldsAreTracked = true; + bool isDef = ((lclVarNode->gtFlags & GTF_VAR_DEF) != 0); for (unsigned i = varDsc.lvFieldLclStart; i < varDsc.lvFieldLclStart + varDsc.lvFieldCnt; ++i) { + LclVarDsc* fieldVarDsc = lvaGetDesc(i); #if !defined(TARGET_64BIT) - if (!varTypeIsLong(lvaTable[i].lvType) || !lvaTable[i].lvPromoted) + if (!varTypeIsLong(fieldVarDsc->lvType) || !fieldVarDsc->lvPromoted) #endif // !defined(TARGET_64BIT) { - noway_assert(lvaTable[i].lvIsStructField); + noway_assert(fieldVarDsc->lvIsStructField); } - if (lvaTable[i].lvTracked) + if (fieldVarDsc->lvTracked) { - const unsigned varIndex = lvaTable[i].lvVarIndex; + const unsigned varIndex = fieldVarDsc->lvVarIndex; noway_assert(varIndex < lvaTrackedCount); - VarSetOps::AddElemD(this, varBit, varIndex); + VarSetOps::AddElemD(this, fieldSet, varIndex); + if (isDef && lclVarNode->IsMultiRegLclVar() && !VarSetOps::IsMember(this, life, varIndex)) + { + // Dead definition. + lclVarNode->AsLclVar()->SetLastUse(i - varDsc.lvFieldLclStart); + } + } + else + { + fieldsAreTracked = false; } } - if (lclVarNode->gtFlags & GTF_VAR_DEF) + + if (isDef) { - VarSetOps::DiffD(this, varBit, keepAliveVars); - VarSetOps::DiffD(this, life, varBit); - return; + VARSET_TP liveFields(VarSetOps::Intersection(this, life, fieldSet)); + VarSetOps::DiffD(this, fieldSet, keepAliveVars); + VarSetOps::DiffD(this, life, fieldSet); + if (fieldsAreTracked && VarSetOps::IsEmpty(this, liveFields)) + { + // None of the fields were live, so this is a dead store. + if (!opts.MinOpts()) + { + // keepAliveVars always stay alive + VARSET_TP keepAliveFields(VarSetOps::Intersection(this, fieldSet, keepAliveVars)); + noway_assert(VarSetOps::IsEmpty(this, keepAliveFields)); + + // Do not consider this store dead if the parent local variable is an address exposed local. + return !varDsc.lvAddrExposed; + } + } + return false; } // This is a use. // Are the variables already known to be alive? - if (VarSetOps::IsSubset(this, varBit, life)) + if (VarSetOps::IsSubset(this, fieldSet, life)) { lclVarNode->gtFlags &= ~GTF_VAR_DEATH; // Since we may now call this multiple times, reset if live. - return; + return false; } // Some variables are being used, and they are not currently live. @@ -1674,18 +1705,19 @@ void Compiler::fgComputeLifeUntrackedLocal(VARSET_TP& life, lclVarNode->gtFlags |= GTF_VAR_DEATH; // Are all the variables becoming alive (in the backwards traversal), or just a subset? - if (!VarSetOps::IsEmptyIntersection(this, varBit, life)) + if (!VarSetOps::IsEmptyIntersection(this, fieldSet, life)) { // Only a subset of the variables are become live; we must record that subset. // (Lack of an entry for "lclVarNode" will be considered to imply all become dead in the // forward traversal.) VARSET_TP* deadVarSet = new (this, CMK_bitset) VARSET_TP; - VarSetOps::AssignNoCopy(this, *deadVarSet, VarSetOps::Diff(this, varBit, life)); + VarSetOps::AssignNoCopy(this, *deadVarSet, VarSetOps::Diff(this, fieldSet, life)); GetPromotedStructDeathVars()->Set(lclVarNode, deadVarSet); } // In any case, all the field vars are now live (in the backwards traversal). - VarSetOps::UnionD(this, life, varBit); + VarSetOps::UnionD(this, life, fieldSet); + return false; } //------------------------------------------------------------------------ @@ -1706,12 +1738,13 @@ bool Compiler::fgComputeLifeLocal(VARSET_TP& life, VARSET_VALARG_TP keepAliveVar assert(lclNum < lvaCount); LclVarDsc& varDsc = lvaTable[lclNum]; + bool isDef = ((lclVarNode->gtFlags & GTF_VAR_DEF) != 0); // Is this a tracked variable? if (varDsc.lvTracked) { /* Is this a definition or use? */ - if (lclVarNode->gtFlags & GTF_VAR_DEF) + if (isDef) { return fgComputeLifeTrackedLocalDef(life, keepAliveVars, varDsc, lclVarNode->AsLclVarCommon()); } @@ -1722,7 +1755,7 @@ bool Compiler::fgComputeLifeLocal(VARSET_TP& life, VARSET_VALARG_TP keepAliveVar } else { - fgComputeLifeUntrackedLocal(life, keepAliveVars, varDsc, lclVarNode->AsLclVarCommon()); + return fgComputeLifeUntrackedLocal(life, keepAliveVars, varDsc, lclVarNode->AsLclVarCommon()); } return false; } @@ -2272,7 +2305,24 @@ bool Compiler::fgRemoveDeadStore(GenTree** pTree, else { // This is an INTERIOR STATEMENT with a dead assignment - remove it - noway_assert(!VarSetOps::IsMember(this, life, varDsc->lvVarIndex)); + // TODO-Cleanup: I'm not sure this assert is valuable; we've already determined this when + // we computed that it was dead. + if (varDsc->lvTracked) + { + noway_assert(!VarSetOps::IsMember(this, life, varDsc->lvVarIndex)); + } + else + { + for (unsigned i = 0; i < varDsc->lvFieldCnt; ++i) + { + unsigned fieldVarNum = varDsc->lvFieldLclStart + i; + { + LclVarDsc* fieldVarDsc = lvaGetDesc(fieldVarNum); + noway_assert(fieldVarDsc->lvTracked && + !VarSetOps::IsMember(this, life, fieldVarDsc->lvVarIndex)); + } + } + } if (sideEffList != nullptr) { diff --git a/src/coreclr/src/jit/lower.cpp b/src/coreclr/src/jit/lower.cpp index 902b8bb28b376..0b42380789126 100644 --- a/src/coreclr/src/jit/lower.cpp +++ b/src/coreclr/src/jit/lower.cpp @@ -296,8 +296,28 @@ GenTree* Lowering::LowerNode(GenTree* node) } case GT_LCL_VAR: - WidenSIMD12IfNecessary(node->AsLclVarCommon()); + { + GenTreeLclVar* lclNode = node->AsLclVar(); + WidenSIMD12IfNecessary(lclNode); + LclVarDsc* varDsc = comp->lvaGetDesc(lclNode); + + // The consumer of this node must check compatibility of the fields. + // This merely checks whether it is possible for this to be a multireg node. + if (lclNode->IsMultiRegLclVar()) + { + if (!varDsc->lvPromoted || + (comp->lvaGetPromotionType(varDsc) != Compiler::PROMOTION_TYPE_INDEPENDENT) || + (varDsc->lvFieldCnt > MAX_MULTIREG_COUNT)) + { + lclNode->ClearMultiReg(); + if (lclNode->TypeIs(TYP_STRUCT)) + { + comp->lvaSetVarDoNotEnregister(lclNode->GetLclNum() DEBUGARG(Compiler::DNER_BlockOp)); + } + } + } break; + } case GT_STORE_LCL_VAR: WidenSIMD12IfNecessary(node->AsLclVarCommon()); @@ -2941,37 +2961,47 @@ void Lowering::LowerRet(GenTreeUnOp* ret) BlockRange().InsertBefore(ret, bitcast); ContainCheckBitCast(bitcast); } - else + else if (ret->TypeGet() != TYP_VOID) { -#ifdef DEBUG - if (ret->TypeGet() != TYP_VOID) + GenTree* retVal = ret->gtGetOp1(); +#if FEATURE_MULTIREG_RET + if (op1->OperIs(GT_LCL_VAR) && varTypeIsStruct(op1)) { - GenTree* retVal = ret->gtGetOp1(); + ReturnTypeDesc retTypeDesc; + LclVarDsc* varDsc = nullptr; + varDsc = comp->lvaGetDesc(op1->AsLclVar()->GetLclNum()); + retTypeDesc.InitializeStructReturnType(comp, varDsc->lvVerTypeInfo.GetClassHandle()); + if (retTypeDesc.GetReturnRegCount() > 1) + { + CheckMultiRegLclVar(op1->AsLclVar(), &retTypeDesc); + } + } + else +#ifdef DEBUG if (varTypeIsStruct(ret->TypeGet()) != varTypeIsStruct(retVal->TypeGet())) + { + if (varTypeIsStruct(ret->TypeGet())) { - if (varTypeIsStruct(ret->TypeGet())) + assert(!comp->compDoOldStructRetyping()); + bool actualTypesMatch = false; + if (genActualType(comp->info.compRetNativeType) == genActualType(retVal->TypeGet())) { - assert(!comp->compDoOldStructRetyping()); - bool actualTypesMatch = false; - if (genActualType(comp->info.compRetNativeType) == genActualType(retVal->TypeGet())) - { - // This could happen if we have retyped op1 as a primitive type during struct promotion, - // check `retypedFieldsMap` for details. - actualTypesMatch = true; - } - bool constStructInit = retVal->IsConstInitVal(); - assert(actualTypesMatch || constStructInit); + // This could happen if we have retyped op1 as a primitive type during struct promotion, + // check `retypedFieldsMap` for details. + actualTypesMatch = true; } - else - { + bool constStructInit = retVal->IsConstInitVal(); + assert(actualTypesMatch || constStructInit); + } + else + { #ifdef FEATURE_SIMD - assert(comp->compDoOldStructRetyping()); - assert(ret->TypeIs(TYP_DOUBLE)); - assert(retVal->TypeIs(TYP_SIMD8)); + assert(comp->compDoOldStructRetyping()); + assert(ret->TypeIs(TYP_DOUBLE)); + assert(retVal->TypeIs(TYP_SIMD8)); #else // !FEATURE_SIMD - unreached(); + unreached(); #endif // !FEATURE_SIMD - } } } #endif // DEBUG @@ -2979,6 +3009,7 @@ void Lowering::LowerRet(GenTreeUnOp* ret) { LowerRetStruct(ret); } +#endif // !FEATURE_MULTIREG_RET } // Method doing PInvokes has exactly one return block unless it has tail calls. @@ -3018,7 +3049,17 @@ void Lowering::LowerStoreLocCommon(GenTreeLclVarCommon* lclStore) } } - if ((lclStore->TypeGet() == TYP_STRUCT) && (src->OperGet() != GT_PHI)) + bool srcIsMultiReg = src->IsMultiRegNode(); + if (srcIsMultiReg || lclStore->IsMultiRegLclVar()) + { + const ReturnTypeDesc* retTypeDesc = nullptr; + if (src->OperIs(GT_CALL)) + { + retTypeDesc = src->AsCall()->GetReturnTypeDesc(); + } + CheckMultiRegLclVar(lclStore->AsLclVar(), retTypeDesc); + } + if (!srcIsMultiReg && (lclStore->TypeGet() == TYP_STRUCT) && (src->OperGet() != GT_PHI)) { if (src->OperGet() == GT_CALL) { @@ -5814,6 +5855,12 @@ bool Lowering::CheckBlock(Compiler* compiler, BasicBlock* block) } #endif +//------------------------------------------------------------------------ +// Lowering::LowerBlock: Lower all the nodes in a BasicBlock +// +// Arguments: +// block - the block to lower. +// void Lowering::LowerBlock(BasicBlock* block) { assert(block == comp->compCurBB); // compCurBB must already be set. @@ -5957,6 +6004,74 @@ bool Lowering::NodesAreEquivalentLeaves(GenTree* tree1, GenTree* tree2) } } +//------------------------------------------------------------------------ +// Lowering::CheckMultiRegLclVar: Check whether a MultiReg GT_LCL_VAR node can +// remain a multi-reg. +// +// Arguments: +// lclNode - the GT_LCL_VAR or GT_STORE_LCL_VAR node. +// retTypeDesc - a return type descriptor either for a call source of a store of +// the local, or for the GT_RETURN consumer of the local. +// +// Notes: +// If retTypeDesc is non-null, this method will check that the fields are compatible. +// Otherwise, it will only check that the lclVar is independently promoted +// (i.e. it is marked lvPromoted and not lvDoNotEnregister). +// +bool Lowering::CheckMultiRegLclVar(GenTreeLclVar* lclNode, const ReturnTypeDesc* retTypeDesc) +{ + bool canEnregister = false; +#if FEATURE_MULTIREG_RET + LclVarDsc* varDsc = comp->lvaGetDesc(lclNode->GetLclNum()); + if ((comp->lvaEnregMultiRegVars) && varDsc->lvPromoted) + { + // We can enregister if we have a promoted struct and all the fields' types match the ABI requirements. + // Note that we don't promote structs with explicit layout, so we don't need to check field offsets, and + // if we have multiple types packed into a single register, we won't have matching reg and field counts, + // so we can tolerate mismatches of integer size. + if (varDsc->lvPromoted && (comp->lvaGetPromotionType(varDsc) == Compiler::PROMOTION_TYPE_INDEPENDENT)) + { + // If we have no retTypeDesc, we only care that it is independently promoted. + if (retTypeDesc == nullptr) + { + canEnregister = true; + } + else + { + unsigned regCount = retTypeDesc->GetReturnRegCount(); + + if (regCount == varDsc->lvFieldCnt) + { + canEnregister = true; + } + } + } + } +#ifdef TARGET_XARCH + // For local stores on XARCH we only handle mismatched src/dest register count for + // calls of SIMD type. If the source was another lclVar similarly promoted, we would + // have broken it into multiple stores. + if (lclNode->OperIs(GT_STORE_LCL_VAR) && !lclNode->gtGetOp1()->OperIs(GT_CALL)) + { + canEnregister = false; + } +#endif // TARGET_XARCH + if (canEnregister) + { + lclNode->SetMultiReg(); + } + else + { + lclNode->ClearMultiReg(); + if (varDsc->lvPromoted && !varDsc->lvDoNotEnregister) + { + comp->lvaSetVarDoNotEnregister(lclNode->GetLclNum() DEBUGARG(Compiler::DNER_BlockOp)); + } + } +#endif + return canEnregister; +} + //------------------------------------------------------------------------ // Containment Analysis //------------------------------------------------------------------------ @@ -6149,10 +6264,13 @@ void Lowering::ContainCheckRet(GenTreeUnOp* ret) // This must be a multi-reg return or an HFA of a single element. assert(varDsc->lvIsMultiRegRet || (varDsc->lvIsHfa() && varTypeIsValidHfaType(varDsc->lvType))); - // Mark var as contained if not enregistrable. + // Mark var as contained if not enregisterable. if (!varTypeIsEnregisterable(op1)) { - MakeSrcContained(ret, op1); + if (!op1->IsMultiRegLclVar()) + { + MakeSrcContained(ret, op1); + } } } } diff --git a/src/coreclr/src/jit/lower.h b/src/coreclr/src/jit/lower.h index 0582cfe61e5d6..5f3e13c9b8313 100644 --- a/src/coreclr/src/jit/lower.h +++ b/src/coreclr/src/jit/lower.h @@ -308,6 +308,7 @@ class Lowering final : public Phase #endif void WidenSIMD12IfNecessary(GenTreeLclVarCommon* node); + bool CheckMultiRegLclVar(GenTreeLclVar* lclNode, const ReturnTypeDesc* retTypeDesc); void LowerStoreLoc(GenTreeLclVarCommon* tree); GenTree* LowerArrElem(GenTree* node); void LowerRotate(GenTree* tree); diff --git a/src/coreclr/src/jit/lsra.cpp b/src/coreclr/src/jit/lsra.cpp index 3bba0bd0783da..df8e9dda6bb1b 100644 --- a/src/coreclr/src/jit/lsra.cpp +++ b/src/coreclr/src/jit/lsra.cpp @@ -153,6 +153,17 @@ void lsraAssignRegToTree(GenTree* tree, regNumber reg, unsigned regIdx) putArg->SetRegNumByIdx(reg, regIdx); } #endif // FEATURE_ARG_SPLIT +#if defined(TARGET_XARCH) && defined(FEATURE_HW_INTRINSICS) + else if (tree->OperIs(GT_HWINTRINSIC)) + { + assert(regIdx == 1); + tree->AsHWIntrinsic()->SetOtherReg(reg); + } +#endif + else if (tree->OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR)) + { + tree->AsLclVar()->SetRegNumByIdx(reg, regIdx); + } else { assert(tree->IsMultiRegCall()); @@ -1725,6 +1736,34 @@ void LinearScan::identifyCandidates() { localVarIntervals[varDsc->lvVarIndex] = nullptr; } + // The current implementation of multi-reg structs that are referenced collectively + // (i.e. by refering to the parent lclVar rather than each field separately) relies + // on all or none of the fields being candidates. + if (varDsc->lvIsStructField) + { + LclVarDsc* parentVarDsc = compiler->lvaGetDesc(varDsc->lvParentLcl); + if (parentVarDsc->lvIsMultiRegRet && !parentVarDsc->lvDoNotEnregister) + { + JITDUMP("Setting multi-reg struct V%02u as not enregisterable:", varDsc->lvParentLcl); + compiler->lvaSetVarDoNotEnregister(varDsc->lvParentLcl DEBUGARG(Compiler::DNER_BlockOp)); + for (unsigned int i = 0; i < parentVarDsc->lvFieldCnt; i++) + { + LclVarDsc* fieldVarDsc = compiler->lvaGetDesc(parentVarDsc->lvFieldLclStart + i); + JITDUMP(" V%02u", parentVarDsc->lvFieldLclStart + i); + if (fieldVarDsc->lvTracked) + { + fieldVarDsc->lvLRACandidate = 0; + localVarIntervals[fieldVarDsc->lvVarIndex] = nullptr; + VarSetOps::RemoveElemD(compiler, registerCandidateVars, fieldVarDsc->lvVarIndex); + JITDUMP("*"); + } + // This is not accurate, but we need a non-zero refCnt for the parent so that it will + // be allocated to the stack. + parentVarDsc->setLvRefCnt(parentVarDsc->lvRefCnt() + fieldVarDsc->lvRefCnt()); + } + JITDUMP("\n"); + } + } continue; } @@ -2148,14 +2187,14 @@ void LinearScan::checkLastUses(BasicBlock* block) { // NOTE: this is a bit of a hack. When extending lifetimes, the "last use" bit will be clear. // This bit, however, would normally be used during resolveLocalRef to set the value of - // GTF_VAR_DEATH on the node for a ref position. If this bit is not set correctly even when + // LastUse on the node for a ref position. If this bit is not set correctly even when // extending lifetimes, the code generator will assert as it expects to have accurate last - // use information. To avoid these asserts, set the GTF_VAR_DEATH bit here. + // use information. To avoid these asserts, set the LastUse bit here. // Note also that extendLifetimes() is an LSRA stress mode, so it will only be true for // Checked or Debug builds, for which this method will be executed. if (tree != nullptr) { - tree->gtFlags |= GTF_VAR_DEATH; + tree->AsLclVar()->SetLastUse(currentRefPosition->multiRegIdx); } } else if (!currentRefPosition->lastUse) @@ -2174,7 +2213,7 @@ void LinearScan::checkLastUses(BasicBlock* block) else if (extendLifetimes() && tree != nullptr) { // NOTE: see the comment above re: the extendLifetimes hack. - tree->gtFlags &= ~GTF_VAR_DEATH; + tree->AsLclVar()->ClearLastUse(currentRefPosition->multiRegIdx); } if (currentRefPosition->refType == RefTypeDef || currentRefPosition->refType == RefTypeDummyDef) @@ -4421,7 +4460,8 @@ void LinearScan::unassignPhysReg(RegRecord* regRec, RefPosition* spillRefPositio // it would be messy and undesirably cause the "bleeding" of LSRA stress modes outside // of LSRA. if (extendLifetimes() && assignedInterval->isLocalVar && RefTypeIsUse(spillRefPosition->refType) && - spillRefPosition->treeNode != nullptr && (spillRefPosition->treeNode->gtFlags & GTF_VAR_DEATH) != 0) + spillRefPosition->treeNode != nullptr && + spillRefPosition->treeNode->AsLclVar()->IsLastUse(spillRefPosition->multiRegIdx)) { dumpLsraAllocationEvent(LSRA_EVENT_SPILL_EXTENDED_LIFETIME, assignedInterval); assignedInterval->isActive = false; @@ -6295,12 +6335,25 @@ void LinearScan::updatePreviousInterval(RegRecord* reg, Interval* interval, Regi // Return Value: // None // +// Note: +// For a multireg node, 'varNum' will be the field local for the given register. +// void LinearScan::writeLocalReg(GenTreeLclVar* lclNode, unsigned varNum, regNumber reg) { - // We don't yet support multireg locals. - assert((lclNode->GetLclNum() == varNum) && !lclNode->IsMultiReg()); - assert(lclNode->GetLclNum() == varNum); - lclNode->SetRegNum(reg); + assert((lclNode->GetLclNum() == varNum) == !lclNode->IsMultiReg()); + if (lclNode->GetLclNum() == varNum) + { + lclNode->SetRegNum(reg); + } + else + { + assert(compiler->lvaEnregMultiRegVars); + LclVarDsc* parentVarDsc = compiler->lvaGetDesc(lclNode->GetLclNum()); + assert(parentVarDsc->lvPromoted); + unsigned regIndex = varNum - parentVarDsc->lvFieldLclStart; + assert(regIndex < MAX_MULTIREG_COUNT); + lclNode->SetRegNumByIdx(reg, regIndex); + } } //----------------------------------------------------------------------------- @@ -6353,7 +6406,7 @@ void LinearScan::resolveLocalRef(BasicBlock* block, GenTreeLclVar* treeNode, Ref interval->recentRefPosition = currentRefPosition; LclVarDsc* varDsc = interval->getLocalVar(compiler); - // NOTE: we set the GTF_VAR_DEATH flag here unless we are extending lifetimes, in which case we write + // NOTE: we set the LastUse flag here unless we are extending lifetimes, in which case we write // this bit in checkLastUses. This is a bit of a hack, but is necessary because codegen requires // accurate last use info that is not reflected in the lastUse bit on ref positions when we are extending // lifetimes. See also the comments in checkLastUses. @@ -6392,7 +6445,9 @@ void LinearScan::resolveLocalRef(BasicBlock* block, GenTreeLclVar* treeNode, Ref } interval->assignedReg = nullptr; interval->physReg = REG_NA; - if (currentRefPosition->refType == RefTypeUse) + // Set this as contained if it is not a multi-reg (we could potentially mark it s contained + // if all uses are from spill, but that adds complexity. + if ((currentRefPosition->refType == RefTypeUse) && !treeNode->IsMultiReg()) { assert(treeNode != nullptr); treeNode->SetContained(); @@ -6454,6 +6509,10 @@ void LinearScan::resolveLocalRef(BasicBlock* block, GenTreeLclVar* treeNode, Ref if (treeNode != nullptr) { treeNode->gtFlags |= GTF_SPILLED; + if (treeNode->IsMultiReg()) + { + treeNode->SetRegSpillFlagByIdx(GTF_SPILLED, currentRefPosition->getMultiRegIdx()); + } if (spillAfter) { if (currentRefPosition->RegOptional()) @@ -6476,6 +6535,10 @@ void LinearScan::resolveLocalRef(BasicBlock* block, GenTreeLclVar* treeNode, Ref else { treeNode->gtFlags |= GTF_SPILL; + if (treeNode->IsMultiReg()) + { + treeNode->SetRegSpillFlagByIdx(GTF_SPILL, currentRefPosition->getMultiRegIdx()); + } } } } @@ -6557,6 +6620,10 @@ void LinearScan::resolveLocalRef(BasicBlock* block, GenTreeLclVar* treeNode, Ref if (treeNode != nullptr) { treeNode->gtFlags |= GTF_SPILL; + if (treeNode->IsMultiReg()) + { + treeNode->SetRegSpillFlagByIdx(GTF_SPILL, currentRefPosition->getMultiRegIdx()); + } } assert(interval->isSpilled); interval->physReg = REG_NA; @@ -6574,6 +6641,10 @@ void LinearScan::resolveLocalRef(BasicBlock* block, GenTreeLclVar* treeNode, Ref if (!currentRefPosition->lastUse) { treeNode->gtFlags |= GTF_SPILLED; + if (treeNode->IsMultiReg()) + { + treeNode->SetRegSpillFlagByIdx(GTF_SPILLED, currentRefPosition->getMultiRegIdx()); + } } } } @@ -9236,12 +9307,15 @@ void RefPosition::dump() { this->getInterval()->tinyDump(); } - if (this->treeNode) { - printf("%s ", treeNode->OpName(treeNode->OperGet())); + printf("%s", treeNode->OpName(treeNode->OperGet())); + if (this->treeNode->IsMultiRegNode()) + { + printf("[%d]", this->multiRegIdx); + } } - printf(FMT_BB " ", this->bbNum); + printf(" " FMT_BB " ", this->bbNum); printf("regmask="); dumpRegMask(registerAssignment); @@ -9498,7 +9572,9 @@ void LinearScan::lsraGetOperandString(GenTree* tree, if (tree->IsMultiRegNode()) { - unsigned regCount = tree->GetMultiRegCount(); + unsigned regCount = tree->IsMultiRegLclVar() + ? compiler->lvaGetDesc(tree->AsLclVar()->GetLclNum())->lvFieldCnt + : tree->GetMultiRegCount(); for (unsigned regIndex = 1; regIndex < regCount; regIndex++) { regNumber reg = tree->GetRegByIndex(regIndex); diff --git a/src/coreclr/src/jit/lsra.h b/src/coreclr/src/jit/lsra.h index 7c4056241daca..97b37b644c6dc 100644 --- a/src/coreclr/src/jit/lsra.h +++ b/src/coreclr/src/jit/lsra.h @@ -1537,6 +1537,9 @@ class LinearScan : public LinearScanInterface pendingDelayFree = false; } + bool isCandidateMultiRegLclVar(GenTreeLclVar* lclNode); + bool checkContainedOrCandidateLclVar(GenTreeLclVar* lclNode); + RefPosition* BuildUse(GenTree* operand, regMaskTP candidates = RBM_NONE, int multiRegIdx = 0); void setDelayFree(RefPosition* use); @@ -1577,6 +1580,7 @@ class LinearScan : public LinearScanInterface int BuildModDiv(GenTree* tree); int BuildIntrinsic(GenTree* tree); void BuildStoreLocDef(GenTreeLclVarCommon* storeLoc, LclVarDsc* varDsc, RefPosition* singleUseRef, int index); + int BuildMultiRegStoreLoc(GenTreeLclVar* storeLoc); int BuildStoreLoc(GenTreeLclVarCommon* tree); int BuildIndir(GenTreeIndir* indirTree); int BuildGCWriteBarrier(GenTree* tree); diff --git a/src/coreclr/src/jit/lsraarm.cpp b/src/coreclr/src/jit/lsraarm.cpp index 165ad929fbc05..7c745589108d3 100644 --- a/src/coreclr/src/jit/lsraarm.cpp +++ b/src/coreclr/src/jit/lsraarm.cpp @@ -219,40 +219,25 @@ int LinearScan::BuildNode(GenTree* tree) switch (tree->OperGet()) { case GT_LCL_VAR: - { - // We handle tracked variables differently from non-tracked ones. If it is tracked, - // we will simply add a use of the tracked variable at its parent/consumer. - // Otherwise, for a use we need to actually add the appropriate references for loading - // or storing the variable. - // - // A tracked variable won't actually get used until the appropriate ancestor tree node - // is processed, unless this is marked "isLocalDefUse" because it is a stack-based argument - // to a call or an orphaned dead node. - // - bool isCandidate = compiler->lvaGetDesc(tree->AsLclVar())->lvLRACandidate; - if (tree->IsRegOptional() && !isCandidate) - { - tree->ClearRegOptional(); - tree->SetContained(); - return 0; - } - if (isCandidate) + // We make a final determination about whether a GT_LCL_VAR is a candidate or contained + // after liveness. In either case we don't build any uses or defs. Otherwise, this is a + // load of a stack-based local into a register and we'll fall through to the general + // local case below. + if (checkContainedOrCandidateLclVar(tree->AsLclVar())) { return 0; } - } __fallthrough; case GT_LCL_FLD: { - GenTreeLclVarCommon* const lclVar = tree->AsLclVarCommon(); - if (lclVar->OperIs(GT_LCL_FLD) && lclVar->AsLclFld()->IsOffsetMisaligned()) + if (tree->OperIs(GT_LCL_FLD) && tree->AsLclFld()->IsOffsetMisaligned()) { - buildInternalIntRegisterDefForNode(lclVar); // to generate address. - buildInternalIntRegisterDefForNode(lclVar); // to move float into an int reg. - if (lclVar->TypeIs(TYP_DOUBLE)) + buildInternalIntRegisterDefForNode(tree); // to generate address. + buildInternalIntRegisterDefForNode(tree); // to move float into an int reg. + if (tree->TypeIs(TYP_DOUBLE)) { - buildInternalIntRegisterDefForNode(lclVar); // to move the second half into an int reg. + buildInternalIntRegisterDefForNode(tree); // to move the second half into an int reg. } buildInternalRegisterUses(); } @@ -262,8 +247,14 @@ int LinearScan::BuildNode(GenTree* tree) } break; - case GT_STORE_LCL_FLD: case GT_STORE_LCL_VAR: + if (tree->IsMultiRegLclVar() && isCandidateMultiRegLclVar(tree->AsLclVar())) + { + dstCount = compiler->lvaGetDesc(tree->AsLclVar()->GetLclNum())->lvFieldCnt; + } + __fallthrough; + + case GT_STORE_LCL_FLD: srcCount = BuildStoreLoc(tree->AsLclVarCommon()); break; @@ -834,7 +825,7 @@ int LinearScan::BuildNode(GenTree* tree) assert((dstCount < 2) || tree->IsMultiRegNode()); assert(isLocalDefUse == (tree->IsValue() && tree->IsUnusedValue())); assert(!tree->IsUnusedValue() || (dstCount != 0)); - assert(dstCount == tree->GetRegisterDstCount()); + assert(dstCount == tree->GetRegisterDstCount(compiler)); return srcCount; } diff --git a/src/coreclr/src/jit/lsraarm64.cpp b/src/coreclr/src/jit/lsraarm64.cpp index c7146d6872af0..bb9ce19a551ac 100644 --- a/src/coreclr/src/jit/lsraarm64.cpp +++ b/src/coreclr/src/jit/lsraarm64.cpp @@ -76,28 +76,14 @@ int LinearScan::BuildNode(GenTree* tree) break; case GT_LCL_VAR: - { - // We handle tracked variables differently from non-tracked ones. If it is tracked, - // we will simply add a use of the tracked variable at its parent/consumer. - // Otherwise, for a use we need to actually add the appropriate references for loading - // or storing the variable. - // - // A tracked variable won't actually get used until the appropriate ancestor tree node - // is processed, unless this is marked "isLocalDefUse" because it is a stack-based argument - // to a call or an orphaned dead node. - // - bool isCandidate = compiler->lvaGetDesc(tree->AsLclVar())->lvLRACandidate; - if (tree->IsRegOptional() && !isCandidate) - { - tree->ClearRegOptional(); - tree->SetContained(); - return 0; - } - if (isCandidate) + // We make a final determination about whether a GT_LCL_VAR is a candidate or contained + // after liveness. In either case we don't build any uses or defs. Otherwise, this is a + // load of a stack-based local into a register and we'll fall through to the general + // local case below. + if (checkContainedOrCandidateLclVar(tree->AsLclVar())) { return 0; } - } __fallthrough; case GT_LCL_FLD: @@ -118,10 +104,14 @@ int LinearScan::BuildNode(GenTree* tree) } break; - case GT_STORE_LCL_FLD: case GT_STORE_LCL_VAR: - srcCount = 1; - assert(dstCount == 0); + if (tree->IsMultiRegLclVar() && isCandidateMultiRegLclVar(tree->AsLclVar())) + { + dstCount = compiler->lvaGetDesc(tree->AsLclVar()->GetLclNum())->lvFieldCnt; + } + __fallthrough; + + case GT_STORE_LCL_FLD: srcCount = BuildStoreLoc(tree->AsLclVarCommon()); break; @@ -775,10 +765,10 @@ int LinearScan::BuildNode(GenTree* tree) isLocalDefUse = true; } // We need to be sure that we've set srcCount and dstCount appropriately - assert((dstCount < 2) || tree->IsMultiRegCall()); + assert((dstCount < 2) || tree->IsMultiRegNode()); assert(isLocalDefUse == (tree->IsValue() && tree->IsUnusedValue())); assert(!tree->IsUnusedValue() || (dstCount != 0)); - assert(dstCount == tree->GetRegisterDstCount()); + assert(dstCount == tree->GetRegisterDstCount(compiler)); return srcCount; } diff --git a/src/coreclr/src/jit/lsrabuild.cpp b/src/coreclr/src/jit/lsrabuild.cpp index c1a5e0b586b40..dc08fc368f577 100644 --- a/src/coreclr/src/jit/lsrabuild.cpp +++ b/src/coreclr/src/jit/lsrabuild.cpp @@ -1218,6 +1218,111 @@ bool LinearScan::buildKillPositionsForNode(GenTree* tree, LsraLocation currentLo return insertedKills; } +//------------------------------------------------------------------------ +// LinearScan::isCandidateMultiRegLclVar: Check whether a MultiReg node should +// remain a candidate MultiReg +// +// Arguments: +// lclNode - the GT_LCL_VAR or GT_STORE_LCL_VAR of interest +// +// Return Value: +// true iff it remains a MultiReg lclVar. +// +// Notes: +// When identifying candidates, the register allocator will only retain +// promoted fields of a multi-reg local as candidates if all of its fields +// are candidates. This is because of the added complexity of dealing with a +// def or use of a multi-reg lclVar when only some of the fields have liveness +// info. +// At the time we determine whether a multi-reg lclVar can still be handled +// as such, we've already completed Lowering, so during the build phase of +// LSRA we have to reset the GTF_VAR_MULTIREG flag if necessary as we visit +// each node. +// +bool LinearScan::isCandidateMultiRegLclVar(GenTreeLclVar* lclNode) +{ + assert(compiler->lvaEnregMultiRegVars && lclNode->IsMultiReg()); + LclVarDsc* varDsc = compiler->lvaGetDesc(lclNode->GetLclNum()); + assert(varDsc->lvPromoted); + bool isMultiReg = (compiler->lvaGetPromotionType(varDsc) == Compiler::PROMOTION_TYPE_INDEPENDENT); + if (!isMultiReg) + { + lclNode->ClearMultiReg(); + } +#ifdef DEBUG + for (unsigned int i = 0; i < varDsc->lvFieldCnt; i++) + { + LclVarDsc* fieldVarDsc = compiler->lvaGetDesc(varDsc->lvFieldLclStart + i); + assert(isCandidateVar(fieldVarDsc) == isMultiReg); + } +#endif // DEBUG + return isMultiReg; +} + +//------------------------------------------------------------------------ +// checkContainedOrCandidateLclVar: Check whether a GT_LCL_VAR node is a +// candidate or contained. +// +// Arguments: +// lclNode - the GT_LCL_VAR or GT_STORE_LCL_VAR of interest +// +// Return Value: +// true if the node remains a candidate or is contained +// false otherwise (i.e. if it will define a register) +// +// Notes: +// We handle candidate variables differently from non-candidate ones. +// If it is a candidate, we will simply add a use of it at its parent/consumer. +// Otherwise, for a use we need to actually add the appropriate references for loading +// or storing the variable. +// +// A candidate lclVar won't actually get used until the appropriate ancestor node +// is processed, unless this is marked "isLocalDefUse" because it is a stack-based argument +// to a call or an orphaned dead node. +// +// Also, because we do containment analysis before we redo dataflow and identify register +// candidates, the containment analysis only uses !lvDoNotEnregister to estimate register +// candidates. +// If there is a lclVar that is estimated during Lowering to be register candidate but turns +// out not to be, if a use was marked regOptional it should now be marked contained instead. +// +bool LinearScan::checkContainedOrCandidateLclVar(GenTreeLclVar* lclNode) +{ + bool isCandidate; + bool makeContained = false; + // We shouldn't be calling this if this node was already contained. + assert(!lclNode->isContained()); + // If we have a multireg local, verify that its fields are still register candidates. + if (lclNode->IsMultiReg()) + { + // Multi-reg uses must support containment, but if we have an actual multi-reg local + // we don't want it to be RegOptional in fixed-use cases, so that we can ensure proper + // liveness modeling (e.g. if one field is in a register required by another field, in + // a RegOptional case we won't handle the conflict properly if we decide not to allocate). + isCandidate = isCandidateMultiRegLclVar(lclNode); + if (isCandidate) + { + assert(!lclNode->IsRegOptional()); + } + else + { + makeContained = true; + } + } + else + { + isCandidate = compiler->lvaGetDesc(lclNode)->lvLRACandidate; + makeContained = !isCandidate && lclNode->IsRegOptional(); + } + if (makeContained) + { + lclNode->ClearRegOptional(); + lclNode->SetContained(); + return true; + } + return isCandidate; +} + //---------------------------------------------------------------------------- // defineNewInternalTemp: Defines a ref position for an internal temp. // @@ -1470,7 +1575,6 @@ void LinearScan::buildUpperVectorRestoreRefPosition(Interval* lclVarInterval, Ls // Returns: // The number of registers defined by `operand`. // -// static int LinearScan::ComputeOperandDstCount(GenTree* operand) { // GT_ARGPLACE is the only non-LIR node that is currently in the trees at this stage, though @@ -1499,7 +1603,7 @@ int LinearScan::ComputeOperandDstCount(GenTree* operand) { // Operands that are values and are not contained consume all of their operands // and produce one or more registers. - return operand->GetRegisterDstCount(); + return operand->GetRegisterDstCount(compiler); } else { @@ -1525,7 +1629,6 @@ int LinearScan::ComputeOperandDstCount(GenTree* operand) // Return Value: // The number of registers available as sources for `node`. // -// static int LinearScan::ComputeAvailableSrcCount(GenTree* node) { int numSources = 0; @@ -1603,12 +1706,6 @@ void LinearScan::buildRefPositionsForNode(GenTree* tree, BasicBlock* block, Lsra int newDefListCount = defList.Count(); int produce = newDefListCount - oldDefListCount; assert((consume == 0) || (ComputeAvailableSrcCount(tree) == consume)); - -#if defined(TARGET_AMD64) - // Multi-reg call node is the only node that could produce multi-reg value - assert(produce <= 1 || (tree->IsMultiRegCall() && produce == MAX_RET_REG_COUNT)); -#endif // TARGET_AMD64 - #endif // DEBUG #ifdef DEBUG @@ -2244,15 +2341,15 @@ void LinearScan::buildIntervals() LIR::Range& blockRange = LIR::AsRange(block); for (GenTree* node : blockRange.NonPhiNodes()) { - // We increment the number position of each tree node by 2 to simplify the logic when there's the case of - // a tree that implicitly does a dual-definition of temps (the long case). In this case it is easier to - // already have an idle spot to handle a dual-def instead of making some messy adjustments if we only - // increment the number position by one. + // We increment the location of each tree node by 2 so that the node definition, if any, + // is at a new location and doesn't interfere with the uses. + // For multi-reg local stores, the 'BuildMultiRegStoreLoc' method will further increment the + // location by 2 for each destination register beyond the first. CLANG_FORMAT_COMMENT_ANCHOR; #ifdef DEBUG node->gtSeqNum = currentLoc; - // In DEBUG, we want to set the gtRegTag to GT_REGTAG_REG, so that subsequent dumps will so the register + // In DEBUG, we want to set the gtRegTag to GT_REGTAG_REG, so that subsequent dumps will show the register // value. // Although this looks like a no-op it sets the tag. node->SetRegNum(node->GetRegNum()); @@ -2818,6 +2915,20 @@ RefPosition* LinearScan::BuildUse(GenTree* operand, regMaskTP candidates, int mu } #if FEATURE_PARTIAL_SIMD_CALLEE_SAVE buildUpperVectorRestoreRefPosition(interval, currentLoc, operand); +#endif + } + else if (operand->IsMultiRegLclVar()) + { + assert(compiler->lvaEnregMultiRegVars); + LclVarDsc* varDsc = compiler->lvaGetDesc(operand->AsLclVar()->GetLclNum()); + LclVarDsc* fieldVarDsc = compiler->lvaGetDesc(varDsc->lvFieldLclStart + multiRegIdx); + interval = getIntervalForLocalVar(fieldVarDsc->lvVarIndex); + if (operand->AsLclVar()->IsLastUse(multiRegIdx)) + { + VarSetOps::RemoveElemD(compiler, currentLiveVars, fieldVarDsc->lvVarIndex); + } +#if FEATURE_PARTIAL_SIMD_CALLEE_SAVE + buildUpperVectorRestoreRefPosition(interval, currentLoc, operand); #endif } else @@ -3034,6 +3145,127 @@ int LinearScan::BuildBinaryUses(GenTreeOp* node, regMaskTP candidates) return srcCount; } +//------------------------------------------------------------------------ +// BuildStoreLocDef: Build a definition RefPosition for a local store +// +// Arguments: +// storeLoc - the local store (GT_STORE_LCL_FLD or GT_STORE_LCL_VAR) +// +// Notes: +// This takes an index to enable building multiple defs for a multi-reg local. +// +void LinearScan::BuildStoreLocDef(GenTreeLclVarCommon* storeLoc, + LclVarDsc* varDsc, + RefPosition* singleUseRef, + int index) +{ + assert(varDsc->lvTracked); + unsigned varIndex = varDsc->lvVarIndex; + Interval* varDefInterval = getIntervalForLocalVar(varIndex); + if ((storeLoc->gtFlags & GTF_VAR_DEATH) == 0) + { + VarSetOps::AddElemD(compiler, currentLiveVars, varIndex); + } + if (singleUseRef != nullptr) + { + Interval* srcInterval = singleUseRef->getInterval(); + if (srcInterval->relatedInterval == nullptr) + { + // Preference the source to the dest, unless this is a non-last-use localVar. + // Note that the last-use info is not correct, but it is a better approximation than preferencing + // the source to the dest, if the source's lifetime extends beyond the dest. + if (!srcInterval->isLocalVar || (singleUseRef->treeNode->gtFlags & GTF_VAR_DEATH) != 0) + { + srcInterval->assignRelatedInterval(varDefInterval); + } + } + else if (!srcInterval->isLocalVar) + { + // Preference the source to dest, if src is not a local var. + srcInterval->assignRelatedInterval(varDefInterval); + } + } + RefPosition* def = + newRefPosition(varDefInterval, currentLoc + 1, RefTypeDef, storeLoc, allRegs(varDsc->TypeGet()), index); + if (varDefInterval->isWriteThru) + { + // We always make write-thru defs reg-optional, as we can store them if they don't + // get a register. + def->regOptional = true; + } +#if FEATURE_PARTIAL_SIMD_CALLEE_SAVE + if (varTypeNeedsPartialCalleeSave(varDefInterval->registerType)) + { + varDefInterval->isPartiallySpilled = false; + } +#endif // FEATURE_PARTIAL_SIMD_CALLEE_SAVE +} + +//------------------------------------------------------------------------ +// BuildMultiRegStoreLoc: Set register requirements for a store of a lclVar +// +// Arguments: +// storeLoc - the multireg local store (GT_STORE_LCL_VAR) +// +// Returns: +// The number of source registers read. +// +int LinearScan::BuildMultiRegStoreLoc(GenTreeLclVar* storeLoc) +{ + GenTree* op1 = storeLoc->gtGetOp1(); + unsigned int dstCount = storeLoc->GetFieldCount(compiler); + unsigned int srcCount = dstCount; + LclVarDsc* varDsc = compiler->lvaGetDesc(storeLoc->GetLclNum()); + + assert(compiler->lvaEnregMultiRegVars); + assert(storeLoc->OperGet() == GT_STORE_LCL_VAR); + bool isMultiRegSrc = op1->IsMultiRegNode(); + // The source must be: + // - a multi-reg source + // - an enregisterable SIMD type, or + // - in-memory local + // + if (isMultiRegSrc) + { + assert(op1->GetMultiRegCount() == srcCount); + } + else if (varTypeIsEnregisterable(op1)) + { + // Create a delay free use, as we'll have to use it to create each field + RefPosition* use = BuildUse(op1, RBM_NONE); + setDelayFree(use); + srcCount = 1; + } + else + { + // Otherwise we must have an in-memory struct lclVar. + // We will just load directly into the register allocated for this lclVar, + // so we don't need to build any uses. + assert(op1->OperIs(GT_LCL_VAR) && op1->isContained() && op1->TypeIs(TYP_STRUCT)); + srcCount = 0; + } + // For multi-reg local stores of multi-reg sources, the code generator will read each source + // register, and then move it, if needed, to the destination register. These nodes have + // 2*N locations where N is the number of registers, so that the liveness can + // be reflected accordingly. + // + for (unsigned int i = 0; i < dstCount; ++i) + { + if (isMultiRegSrc) + { + BuildUse(op1, RBM_NONE, i); + } + LclVarDsc* fieldVarDsc = compiler->lvaGetDesc(varDsc->lvFieldLclStart + i); + assert(isCandidateVar(fieldVarDsc)); + BuildStoreLocDef(storeLoc, fieldVarDsc, nullptr, i); + if (isMultiRegSrc && (i < (dstCount - 1))) + { + currentLoc += 2; + } + } + return srcCount; +} + //------------------------------------------------------------------------ // BuildStoreLoc: Set register requirements for a store of a lclVar // @@ -3042,7 +3274,7 @@ int LinearScan::BuildBinaryUses(GenTreeOp* node, regMaskTP candidates) // // Notes: // This involves: -// - Setting the appropriate candidates for a store of a multi-reg call return value. +// - Setting the appropriate candidates. // - Handling of contained immediates. // - Requesting an internal register for SIMD12 stores. // @@ -3053,6 +3285,11 @@ int LinearScan::BuildStoreLoc(GenTreeLclVarCommon* storeLoc) RefPosition* singleUseRef = nullptr; LclVarDsc* varDsc = compiler->lvaGetDesc(storeLoc->GetLclNum()); + if (storeLoc->IsMultiRegLclVar()) + { + return BuildMultiRegStoreLoc(storeLoc->AsLclVar()); + } + // First, define internal registers. #ifdef FEATURE_SIMD RefPosition* internalFloatDef = nullptr; @@ -3065,17 +3302,12 @@ int LinearScan::BuildStoreLoc(GenTreeLclVarCommon* storeLoc) // Second, use source registers. - if (op1->IsMultiRegCall()) + if (op1->IsMultiRegNode()) { - // This is the case of var = call where call is returning - // a value in multiple return registers. - // Must be a store lclvar. + // This is the case where the source produces multiple registers. + // This must be a store lclvar. assert(storeLoc->OperGet() == GT_STORE_LCL_VAR); - - // srcCount = number of registers in which the value is returned by call - const GenTreeCall* call = op1->AsCall(); - const ReturnTypeDesc* retTypeDesc = call->GetReturnTypeDesc(); - srcCount = retTypeDesc->GetReturnRegCount(); + srcCount = op1->GetMultiRegCount(); for (int i = 0; i < srcCount; ++i) { @@ -3095,19 +3327,11 @@ int LinearScan::BuildStoreLoc(GenTreeLclVarCommon* storeLoc) #ifndef TARGET_64BIT else if (varTypeIsLong(op1)) { - if (op1->OperIs(GT_MUL_LONG)) - { - srcCount = 2; - BuildUse(op1, allRegs(TYP_INT), 0); - BuildUse(op1, allRegs(TYP_INT), 1); - } - else - { - assert(op1->OperIs(GT_LONG)); - assert(op1->isContained() && !op1->gtGetOp1()->isContained() && !op1->gtGetOp2()->isContained()); - srcCount = BuildBinaryUses(op1->AsOp()); - assert(srcCount == 2); - } + // GT_MUL_LONG is handled by the IsMultiRegNode case above. + assert(op1->OperIs(GT_LONG)); + assert(op1->isContained() && !op1->gtGetOp1()->isContained() && !op1->gtGetOp2()->isContained()); + srcCount = BuildBinaryUses(op1->AsOp()); + assert(srcCount == 2); } #endif // !TARGET_64BIT else if (op1->isContained()) @@ -3163,46 +3387,7 @@ int LinearScan::BuildStoreLoc(GenTreeLclVarCommon* storeLoc) // Add the lclVar to currentLiveVars (if it will remain live) if (isCandidateVar(varDsc)) { - assert(varDsc->lvTracked); - unsigned varIndex = varDsc->lvVarIndex; - Interval* varDefInterval = getIntervalForLocalVar(varIndex); - if ((storeLoc->gtFlags & GTF_VAR_DEATH) == 0) - { - VarSetOps::AddElemD(compiler, currentLiveVars, varIndex); - } - if (singleUseRef != nullptr) - { - Interval* srcInterval = singleUseRef->getInterval(); - if (srcInterval->relatedInterval == nullptr) - { - // Preference the source to the dest, unless this is a non-last-use localVar. - // Note that the last-use info is not correct, but it is a better approximation than preferencing - // the source to the dest, if the source's lifetime extends beyond the dest. - if (!srcInterval->isLocalVar || (singleUseRef->treeNode->gtFlags & GTF_VAR_DEATH) != 0) - { - srcInterval->assignRelatedInterval(varDefInterval); - } - } - else if (!srcInterval->isLocalVar) - { - // Preference the source to dest, if src is not a local var. - srcInterval->assignRelatedInterval(varDefInterval); - } - } - RefPosition* def = - newRefPosition(varDefInterval, currentLoc + 1, RefTypeDef, storeLoc, allRegs(storeLoc->TypeGet())); - if (varDefInterval->isWriteThru) - { - // We always make write-thru defs reg-optional, as we can store them if they don't - // get a register. - def->regOptional = true; - } -#if FEATURE_PARTIAL_SIMD_CALLEE_SAVE - if (varTypeNeedsPartialCalleeSave(varDefInterval->registerType)) - { - varDefInterval->isPartiallySpilled = false; - } -#endif // FEATURE_PARTIAL_SIMD_CALLEE_SAVE + BuildStoreLocDef(storeLoc, varDsc, singleUseRef, 0); } return srcCount; @@ -3264,7 +3449,7 @@ int LinearScan::BuildReturn(GenTree* tree) #if FEATURE_MULTIREG_RET #ifdef TARGET_ARM64 - if (varTypeIsSIMD(tree)) + if (varTypeIsSIMD(tree) && !op1->IsMultiRegLclVar()) { useCandidates = allSIMDRegs(); BuildUse(op1, useCandidates); @@ -3274,21 +3459,45 @@ int LinearScan::BuildReturn(GenTree* tree) if (varTypeIsStruct(tree)) { - // op1 has to be either an lclvar or a multi-reg returning call - if (op1->OperGet() == GT_LCL_VAR) + // op1 has to be either a lclvar or a multi-reg returning call + if ((op1->OperGet() == GT_LCL_VAR) && !op1->IsMultiRegLclVar()) { BuildUse(op1, useCandidates); } else { - noway_assert(op1->IsMultiRegCall()); + noway_assert(op1->IsMultiRegCall() || op1->IsMultiRegLclVar()); - const ReturnTypeDesc* retTypeDesc = op1->AsCall()->GetReturnTypeDesc(); - const int srcCount = retTypeDesc->GetReturnRegCount(); - useCandidates = retTypeDesc->GetABIReturnRegs(); + int srcCount; + const ReturnTypeDesc* pRetTypeDesc; + if (op1->OperIs(GT_CALL)) + { + pRetTypeDesc = op1->AsCall()->GetReturnTypeDesc(); + } + else + { + assert(compiler->lvaEnregMultiRegVars); + LclVarDsc* varDsc = compiler->lvaGetDesc(op1->AsLclVar()->GetLclNum()); + ReturnTypeDesc retTypeDesc; + retTypeDesc.InitializeStructReturnType(compiler, varDsc->lvVerTypeInfo.GetClassHandle()); + pRetTypeDesc = &retTypeDesc; + assert(compiler->lvaGetDesc(op1->AsLclVar()->GetLclNum())->lvFieldCnt == + retTypeDesc.GetReturnRegCount()); + } + srcCount = pRetTypeDesc->GetReturnRegCount(); for (int i = 0; i < srcCount; i++) { - BuildUse(op1, useCandidates, i); + // We will build uses of the type of the operand registers/fields, and the codegen + // for return will move as needed. + if (!op1->OperIs(GT_LCL_VAR) || (regType(op1->AsLclVar()->GetFieldTypeByIndex(compiler, i)) == + regType(pRetTypeDesc->GetReturnRegType(i)))) + { + BuildUse(op1, genRegMask(pRetTypeDesc->GetABIReturnReg(i)), i); + } + else + { + BuildUse(op1, RBM_NONE, i); + } } return srcCount; } diff --git a/src/coreclr/src/jit/lsraxarch.cpp b/src/coreclr/src/jit/lsraxarch.cpp index 1239ca23c7bbe..7dd7a68efc71b 100644 --- a/src/coreclr/src/jit/lsraxarch.cpp +++ b/src/coreclr/src/jit/lsraxarch.cpp @@ -83,33 +83,14 @@ int LinearScan::BuildNode(GenTree* tree) break; case GT_LCL_VAR: - { - // We handle tracked variables differently from non-tracked ones. If it is tracked, - // we will simply add a use of the tracked variable at its parent/consumer. - // Otherwise, for a use we need to actually add the appropriate references for loading - // or storing the variable. - // - // A tracked variable won't actually get used until the appropriate ancestor tree node - // is processed, unless this is marked "isLocalDefUse" because it is a stack-based argument - // to a call or an orphaned dead node. - // - // Because we do containment analysis before we redo dataflow and identify register - // candidates, the containment analysis only uses !lvDoNotEnregister to estimate register - // candidates. - // If there is a lclVar that is estimated to be register candidate but - // is not, if they were marked regOptional they should now be marked contained instead. - bool isCandidate = compiler->lvaGetDesc(tree->AsLclVar())->lvLRACandidate; - if (tree->IsRegOptional() && !isCandidate) - { - tree->ClearRegOptional(); - tree->SetContained(); - return 0; - } - if (isCandidate) + // We make a final determination about whether a GT_LCL_VAR is a candidate or contained + // after liveness. In either case we don't build any uses or defs. Otherwise, this is a + // load of a stack-based local into a register and we'll fall through to the general + // local case below. + if (checkContainedOrCandidateLclVar(tree->AsLclVar())) { return 0; } - } __fallthrough; case GT_LCL_FLD: @@ -132,6 +113,10 @@ int LinearScan::BuildNode(GenTree* tree) case GT_STORE_LCL_FLD: case GT_STORE_LCL_VAR: + if (tree->IsMultiRegLclVar() && isCandidateMultiRegLclVar(tree->AsLclVar())) + { + dstCount = compiler->lvaGetDesc(tree->AsLclVar()->GetLclNum())->lvFieldCnt; + } srcCount = BuildStoreLoc(tree->AsLclVarCommon()); break; @@ -707,7 +692,7 @@ int LinearScan::BuildNode(GenTree* tree) assert((dstCount < 2) || ((dstCount == 2) && tree->IsMultiRegNode())); assert(isLocalDefUse == (tree->IsValue() && tree->IsUnusedValue())); assert(!tree->IsUnusedValue() || (dstCount != 0)); - assert(dstCount == tree->GetRegisterDstCount()); + assert(dstCount == tree->GetRegisterDstCount(compiler)); return srcCount; } diff --git a/src/coreclr/src/jit/morph.cpp b/src/coreclr/src/jit/morph.cpp index 68ac3f79bc18a..44b79e600335f 100644 --- a/src/coreclr/src/jit/morph.cpp +++ b/src/coreclr/src/jit/morph.cpp @@ -3672,7 +3672,6 @@ GenTreeCall* Compiler::fgMorphArgs(GenTreeCall* call) // actually passed in registers. if (argEntry->isPassedInRegisters()) { - assert(argEntry->structDesc.passedInRegisters); if (argObj->OperIs(GT_OBJ)) { if (passingSize != structSize) @@ -10730,8 +10729,18 @@ GenTree* Compiler::fgMorphCopyBlock(GenTree* tree) { if (!destLclVar->lvRegStruct || (destLclVar->lvType != dest->TypeGet())) { - // Mark it as DoNotEnregister. - lvaSetVarDoNotEnregister(destLclNum DEBUGARG(DNER_BlockOp)); + if (!dest->IsMultiRegLclVar() || (blockWidth != destLclVar->lvExactSize) || + (destLclVar->lvCustomLayout && destLclVar->lvContainsHoles)) + { + // Mark it as DoNotEnregister. + lvaSetVarDoNotEnregister(destLclNum DEBUGARG(DNER_BlockOp)); + } + else if (dest->IsMultiRegLclVar()) + { + // Handle this as lvIsMultiRegRet; this signals to SSA that it can't consider these fields + // SSA candidates (we don't have a way to represent multiple SSANums on MultiRegLclVar nodes). + destLclVar->lvIsMultiRegRet = true; + } } } diff --git a/src/coreclr/src/jit/treelifeupdater.cpp b/src/coreclr/src/jit/treelifeupdater.cpp index d1a574c071fe8..9b93ed7511545 100644 --- a/src/coreclr/src/jit/treelifeupdater.cpp +++ b/src/coreclr/src/jit/treelifeupdater.cpp @@ -19,6 +19,150 @@ TreeLifeUpdater::TreeLifeUpdater(Compiler* compiler) { } +//------------------------------------------------------------------------ +// UpdateLifeFieldVar: Update live sets for only the given field of a multi-reg LclVar node. +// +// Arguments: +// lclNode - the GT_LCL_VAR node. +// multiRegIndex - the index of the field being updated. +// +// Return Value: +// Returns true iff the variable needs to be spilled. +// +// Notes: +// This method need only be used when the fields are dying or going live at different times, +// e.g. when I ready the 0th field/reg of one node and define the 0th field/reg of another +// before reading the subsequent fields/regs. +// +template +bool TreeLifeUpdater::UpdateLifeFieldVar(GenTreeLclVar* lclNode, unsigned multiRegIndex) +{ + LclVarDsc* parentVarDsc = compiler->lvaGetDesc(lclNode); + assert(parentVarDsc->lvPromoted && (multiRegIndex < parentVarDsc->lvFieldCnt) && lclNode->IsMultiReg() && + compiler->lvaEnregMultiRegVars); + unsigned fieldVarNum = parentVarDsc->lvFieldLclStart + multiRegIndex; + LclVarDsc* fldVarDsc = compiler->lvaGetDesc(fieldVarNum); + assert(fldVarDsc->lvTracked); + unsigned fldVarIndex = fldVarDsc->lvVarIndex; + assert((lclNode->gtFlags & GTF_VAR_USEASG) == 0); + + VarSetOps::Assign(compiler, newLife, compiler->compCurLife); + bool isBorn = ((lclNode->gtFlags & GTF_VAR_DEF) != 0); + bool isDying = !isBorn && lclNode->IsLastUse(multiRegIndex); + // GTF_SPILL will be set if any registers need to be spilled. + unsigned spillFlags = (lclNode->gtFlags & lclNode->GetRegSpillFlagByIdx(multiRegIndex)); + bool spill = ((spillFlags & GTF_SPILL) != 0); + bool isInMemory = false; + + if (isBorn || isDying) + { + if (ForCodeGen) + { + regNumber reg = lclNode->GetRegNumByIdx(multiRegIndex); + bool isInReg = fldVarDsc->lvIsInReg() && reg != REG_NA; + isInMemory = !isInReg || fldVarDsc->lvLiveInOutOfHndlr; + if (isInReg) + { + if (isBorn) + { + compiler->codeGen->genUpdateVarReg(fldVarDsc, lclNode, multiRegIndex); + } + compiler->codeGen->genUpdateRegLife(fldVarDsc, isBorn, isDying DEBUGARG(lclNode)); + } + } + // First, update the live set + if (isDying) + { + VarSetOps::RemoveElemD(compiler, newLife, fldVarIndex); + } + else + { + VarSetOps::AddElemD(compiler, newLife, fldVarIndex); + } + } + + if (!VarSetOps::Equal(compiler, compiler->compCurLife, newLife)) + { +#ifdef DEBUG + if (compiler->verbose) + { + printf("\t\t\t\t\t\t\tLive vars: "); + dumpConvertedVarSet(compiler, compiler->compCurLife); + printf(" => "); + dumpConvertedVarSet(compiler, newLife); + printf("\n"); + } +#endif // DEBUG + + VarSetOps::Assign(compiler, compiler->compCurLife, newLife); + + if (ForCodeGen) + { + // Only add vars to the gcInfo.gcVarPtrSetCur if they are currently on stack, since the + // gcInfo.gcTrkStkPtrLcls + // includes all TRACKED vars that EVER live on the stack (i.e. are not always in a register). + VarSetOps::Assign(compiler, gcTrkStkDeltaSet, compiler->codeGen->gcInfo.gcTrkStkPtrLcls); + if (isInMemory && VarSetOps::IsMember(compiler, gcTrkStkDeltaSet, fldVarIndex)) + { +#ifdef DEBUG + if (compiler->verbose) + { + printf("\t\t\t\t\t\t\tGCvars: "); + dumpConvertedVarSet(compiler, compiler->codeGen->gcInfo.gcVarPtrSetCur); + printf(" => "); + } +#endif // DEBUG + + if (isBorn) + { + VarSetOps::AddElemD(compiler, compiler->codeGen->gcInfo.gcVarPtrSetCur, fldVarIndex); + } + else + { + VarSetOps::RemoveElemD(compiler, compiler->codeGen->gcInfo.gcVarPtrSetCur, fldVarIndex); + } + +#ifdef DEBUG + if (compiler->verbose) + { + dumpConvertedVarSet(compiler, compiler->codeGen->gcInfo.gcVarPtrSetCur); + printf("\n"); + } +#endif // DEBUG + } + +#ifdef USING_VARIABLE_LIVE_RANGE + // For each of the LclVarDsc that are reporting change, variable or fields + compiler->codeGen->getVariableLiveKeeper()->siStartOrCloseVariableLiveRange(fldVarDsc, fieldVarNum, isBorn, + isDying); +#endif // USING_VARIABLE_LIVE_RANGE + +#ifdef USING_SCOPE_INFO + compiler->codeGen->siUpdate(); +#endif // USING_SCOPE_INFO + } + } + + if (ForCodeGen && spill) + { + if (VarSetOps::IsMember(compiler, compiler->codeGen->gcInfo.gcTrkStkPtrLcls, fldVarIndex)) + { + if (!VarSetOps::IsMember(compiler, compiler->codeGen->gcInfo.gcVarPtrSetCur, fldVarIndex)) + { + VarSetOps::AddElemD(compiler, compiler->codeGen->gcInfo.gcVarPtrSetCur, fldVarIndex); +#ifdef DEBUG + if (compiler->verbose) + { + printf("\t\t\t\t\t\t\tVar V%02u becoming live\n", fieldVarNum); + } +#endif // DEBUG + } + } + return true; + } + return false; +} + //------------------------------------------------------------------------ // UpdateLifeVar: Update live sets for a given tree. // @@ -128,7 +272,35 @@ void TreeLifeUpdater::UpdateLifeVar(GenTree* tree) } else if (ForCodeGen && lclVarTree->IsMultiRegLclVar()) { - assert(!"MultiRegLclVars not yet supported"); + assert(varDsc->lvPromoted && compiler->lvaEnregMultiRegVars); + unsigned firstFieldVarNum = varDsc->lvFieldLclStart; + for (unsigned i = 0; i < varDsc->lvFieldCnt; ++i) + { + LclVarDsc* fldVarDsc = &(compiler->lvaTable[firstFieldVarNum + i]); + noway_assert(fldVarDsc->lvIsStructField); + assert(fldVarDsc->lvTracked); + unsigned fldVarIndex = fldVarDsc->lvVarIndex; + regNumber reg = lclVarTree->AsLclVar()->GetRegNumByIdx(i); + bool isInReg = fldVarDsc->lvIsInReg() && reg != REG_NA; + bool isInMemory = !isInReg || fldVarDsc->lvLiveInOutOfHndlr; + bool isFieldDying = lclVarTree->AsLclVar()->IsLastUse(i); + if ((isBorn && !isFieldDying) || (!isBorn && isFieldDying)) + { + VarSetOps::AddElemD(compiler, varDeltaSet, fldVarIndex); + if (isInMemory) + { + VarSetOps::AddElemD(compiler, stackVarDeltaSet, fldVarIndex); + } + } + if (isInReg) + { + if (isBorn) + { + compiler->codeGen->genUpdateVarReg(fldVarDsc, tree, i); + } + compiler->codeGen->genUpdateRegLife(fldVarDsc, isBorn, isFieldDying DEBUGARG(tree)); + } + } } else if (varDsc->lvPromoted) { diff --git a/src/coreclr/src/jit/treelifeupdater.h b/src/coreclr/src/jit/treelifeupdater.h index c324192c0926b..80095f874f1c2 100644 --- a/src/coreclr/src/jit/treelifeupdater.h +++ b/src/coreclr/src/jit/treelifeupdater.h @@ -15,6 +15,7 @@ class TreeLifeUpdater public: TreeLifeUpdater(Compiler* compiler); void UpdateLife(GenTree* tree); + bool UpdateLifeFieldVar(GenTreeLclVar* lclNode, unsigned multiRegIndex); private: void UpdateLifeVar(GenTree* tree);