diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index 145c074b8fc68..d099fe192fc38 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -1648,7 +1648,6 @@ void CodeGen::genConsumeRegs(GenTree* tree) // Update the life of the lcl var. genUpdateLife(tree); } -#ifdef TARGET_XARCH #ifdef FEATURE_HW_INTRINSICS else if (tree->OperIs(GT_HWINTRINSIC)) { @@ -1656,7 +1655,6 @@ void CodeGen::genConsumeRegs(GenTree* tree) genConsumeMultiOpOperands(hwintrinsic); } #endif // FEATURE_HW_INTRINSICS -#endif // TARGET_XARCH else if (tree->OperIs(GT_BITCAST, GT_NEG, GT_CAST, GT_LSH, GT_RSH, GT_RSZ, GT_ROR, GT_BSWAP, GT_BSWAP16)) { genConsumeRegs(tree->gtGetOp1()); diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 3c19969d0c90d..474ade28e07ed 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -3473,6 +3473,7 @@ class Compiler #if defined(TARGET_ARM64) GenTree* gtNewSimdConvertVectorToMaskNode(var_types type, GenTree* node, CorInfoType simdBaseJitType, unsigned simdSize); GenTree* gtNewSimdConvertMaskToVectorNode(GenTreeHWIntrinsic* node, var_types type); + GenTree* gtNewSimdAllTrueMaskNode(CorInfoType simdBaseJitType, unsigned simdSize); #endif //------------------------------------------------------------------------ diff --git a/src/coreclr/jit/emitloongarch64.h b/src/coreclr/jit/emitloongarch64.h index 135f9cf400673..afaca5b2ba04b 100644 --- a/src/coreclr/jit/emitloongarch64.h +++ b/src/coreclr/jit/emitloongarch64.h @@ -333,7 +333,7 @@ enum EmitCallType EC_FUNC_TOKEN, // Direct call to a helper/static/nonvirtual/global method // EC_FUNC_TOKEN_INDIR, // Indirect call to a helper/static/nonvirtual/global method - // EC_FUNC_ADDR, // Direct call to an absolute address + // EC_FUNC_ADDR, // Direct call to an absolute address EC_INDIR_R, // Indirect call via register diff --git a/src/coreclr/jit/emitriscv64.h b/src/coreclr/jit/emitriscv64.h index 07e603a70afb7..262f44c9ac4bb 100644 --- a/src/coreclr/jit/emitriscv64.h +++ b/src/coreclr/jit/emitriscv64.h @@ -310,7 +310,7 @@ enum EmitCallType EC_FUNC_TOKEN, // Direct call to a helper/static/nonvirtual/global method // EC_FUNC_TOKEN_INDIR, // Indirect call to a helper/static/nonvirtual/global method - // EC_FUNC_ADDR, // Direct call to an absolute address + // EC_FUNC_ADDR, // Direct call to an absolute address // EC_FUNC_VIRTUAL, // Call to a virtual method (using the vtable) EC_INDIR_R, // Indirect call via register diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 878c02ac1603d..5a6dc9a856f86 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -18012,7 +18012,7 @@ bool GenTree::canBeContained() const } else if (OperIsHWIntrinsic() && !isContainableHWIntrinsic()) { - return isEvexEmbeddedMaskingCompatibleHWIntrinsic(); + return isEmbeddedMaskingCompatibleHWIntrinsic(); } return true; @@ -19909,24 +19909,26 @@ bool GenTree::isEvexCompatibleHWIntrinsic() const } //------------------------------------------------------------------------ -// isEvexEmbeddedMaskingCompatibleHWIntrinsic: Checks if the intrinsic is compatible +// isEmbeddedMaskingCompatibleHWIntrinsic : Checks if the intrinsic is compatible // with the EVEX embedded masking form for its intended lowering instruction. // // Return Value: // true if the intrisic node lowering instruction has an EVEX embedded masking // -bool GenTree::isEvexEmbeddedMaskingCompatibleHWIntrinsic() const +bool GenTree::isEmbeddedMaskingCompatibleHWIntrinsic() const { -#if defined(TARGET_XARCH) if (OperIsHWIntrinsic()) { +#if defined(TARGET_XARCH) // TODO-AVX512F-CQ: Expand this to the full set of APIs and make it table driven // using IsEmbMaskingCompatible. For now, however, limit it to some explicit ids // for prototyping purposes. return (AsHWIntrinsic()->GetHWIntrinsicId() == NI_AVX512F_Add); +#elif defined(TARGET_ARM64) + return HWIntrinsicInfo::IsEmbeddedMaskedOperation(AsHWIntrinsic()->GetHWIntrinsicId()) || + HWIntrinsicInfo::IsOptionalEmbeddedMaskedOperation(AsHWIntrinsic()->GetHWIntrinsicId()); +#endif } -#endif // TARGET_XARCH - return false; } diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index c636912180952..079174f67e939 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -557,9 +557,9 @@ enum GenTreeFlags : unsigned int GTF_MDARRLOWERBOUND_NONFAULTING = 0x20000000, // GT_MDARR_LOWER_BOUND -- An MD array lower bound operation that cannot fault. Same as GT_IND_NONFAULTING. -#if defined(TARGET_XARCH) && defined(FEATURE_HW_INTRINSICS) +#ifdef FEATURE_HW_INTRINSICS GTF_HW_EM_OP = 0x10000000, // GT_HWINTRINSIC -- node is used as an operand to an embedded mask -#endif // TARGET_XARCH && FEATURE_HW_INTRINSICS +#endif // FEATURE_HW_INTRINSICS }; inline constexpr GenTreeFlags operator ~(GenTreeFlags a) @@ -1465,7 +1465,7 @@ struct GenTree bool isContainableHWIntrinsic() const; bool isRMWHWIntrinsic(Compiler* comp); bool isEvexCompatibleHWIntrinsic() const; - bool isEvexEmbeddedMaskingCompatibleHWIntrinsic() const; + bool isEmbeddedMaskingCompatibleHWIntrinsic() const; #else bool isCommutativeHWIntrinsic() const { @@ -1487,7 +1487,7 @@ struct GenTree return false; } - bool isEvexEmbeddedMaskingCompatibleHWIntrinsic() const + bool isEmbeddedMaskingCompatibleHWIntrinsic() const { return false; } @@ -2226,7 +2226,7 @@ struct GenTree gtFlags &= ~GTF_ICON_HDL_MASK; } -#if defined(TARGET_XARCH) && defined(FEATURE_HW_INTRINSICS) +#ifdef FEATURE_HW_INTRINSICS bool IsEmbMaskOp() { @@ -2240,7 +2240,7 @@ struct GenTree gtFlags |= GTF_HW_EM_OP; } -#endif // TARGET_XARCH && FEATURE_HW_INTRINSICS +#endif // FEATURE_HW_INTRINSICS static bool HandleKindDataIsInvariant(GenTreeFlags flags); diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index 53970ef4a7460..402cae99b3f63 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -1396,6 +1396,36 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, GenTree* op3 = nullptr; GenTree* op4 = nullptr; + switch (numArgs) + { + case 4: + op4 = getArgForHWIntrinsic(sigReader.GetOp4Type(), sigReader.op4ClsHnd); + op4 = addRangeCheckIfNeeded(intrinsic, op4, mustExpand, immLowerBound, immUpperBound); + op3 = getArgForHWIntrinsic(sigReader.GetOp3Type(), sigReader.op3ClsHnd); + op2 = getArgForHWIntrinsic(sigReader.GetOp2Type(), sigReader.op2ClsHnd); + op1 = getArgForHWIntrinsic(sigReader.GetOp1Type(), sigReader.op1ClsHnd); + break; + + case 3: + op3 = getArgForHWIntrinsic(sigReader.GetOp3Type(), sigReader.op3ClsHnd); + op2 = getArgForHWIntrinsic(sigReader.GetOp2Type(), sigReader.op2ClsHnd); + op1 = getArgForHWIntrinsic(sigReader.GetOp1Type(), sigReader.op1ClsHnd); + break; + + case 2: + op2 = getArgForHWIntrinsic(sigReader.GetOp2Type(), sigReader.op2ClsHnd); + op2 = addRangeCheckIfNeeded(intrinsic, op2, mustExpand, immLowerBound, immUpperBound); + op1 = getArgForHWIntrinsic(sigReader.GetOp1Type(), sigReader.op1ClsHnd); + break; + + case 1: + op1 = getArgForHWIntrinsic(sigReader.GetOp1Type(), sigReader.op1ClsHnd); + break; + + default: + break; + } + switch (numArgs) { case 0: @@ -1407,8 +1437,6 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, case 1: { - op1 = getArgForHWIntrinsic(sigReader.GetOp1Type(), sigReader.op1ClsHnd); - if ((category == HW_Category_MemoryLoad) && op1->OperIs(GT_CAST)) { // Although the API specifies a pointer, if what we have is a BYREF, that's what @@ -1467,10 +1495,6 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, case 2: { - op2 = getArgForHWIntrinsic(sigReader.GetOp2Type(), sigReader.op2ClsHnd); - op2 = addRangeCheckIfNeeded(intrinsic, op2, mustExpand, immLowerBound, immUpperBound); - op1 = getArgForHWIntrinsic(sigReader.GetOp1Type(), sigReader.op1ClsHnd); - retNode = isScalar ? gtNewScalarHWIntrinsicNode(nodeRetType, op1, op2, intrinsic) : gtNewSimdHWIntrinsicNode(nodeRetType, op1, op2, intrinsic, simdBaseJitType, simdSize); @@ -1524,10 +1548,6 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, case 3: { - op3 = getArgForHWIntrinsic(sigReader.GetOp3Type(), sigReader.op3ClsHnd); - op2 = getArgForHWIntrinsic(sigReader.GetOp2Type(), sigReader.op2ClsHnd); - op1 = getArgForHWIntrinsic(sigReader.GetOp1Type(), sigReader.op1ClsHnd); - #ifdef TARGET_ARM64 if (intrinsic == NI_AdvSimd_LoadAndInsertScalar) { @@ -1569,12 +1589,6 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, case 4: { - op4 = getArgForHWIntrinsic(sigReader.GetOp4Type(), sigReader.op4ClsHnd); - op4 = addRangeCheckIfNeeded(intrinsic, op4, mustExpand, immLowerBound, immUpperBound); - op3 = getArgForHWIntrinsic(sigReader.GetOp3Type(), sigReader.op3ClsHnd); - op2 = getArgForHWIntrinsic(sigReader.GetOp2Type(), sigReader.op2ClsHnd); - op1 = getArgForHWIntrinsic(sigReader.GetOp1Type(), sigReader.op1ClsHnd); - assert(!isScalar); retNode = gtNewSimdHWIntrinsicNode(nodeRetType, op1, op2, op3, op4, intrinsic, simdBaseJitType, simdSize); @@ -1591,10 +1605,22 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, } #if defined(TARGET_ARM64) - if (HWIntrinsicInfo::IsMaskedOperation(intrinsic)) + if (HWIntrinsicInfo::IsExplicitMaskedOperation(intrinsic)) { assert(numArgs > 0); GenTree* op1 = retNode->AsHWIntrinsic()->Op(1); + if (intrinsic == NI_Sve_ConditionalSelect) + { + if (op1->IsVectorAllBitsSet()) + { + return retNode->AsHWIntrinsic()->Op(2); + } + else if (op1->IsVectorZero()) + { + return retNode->AsHWIntrinsic()->Op(3); + } + } + if (!varTypeIsMask(op1)) { // Op1 input is a vector. HWInstrinsic requires a mask. diff --git a/src/coreclr/jit/hwintrinsic.h b/src/coreclr/jit/hwintrinsic.h index 5ca302e126f32..e7bd08d5cb33d 100644 --- a/src/coreclr/jit/hwintrinsic.h +++ b/src/coreclr/jit/hwintrinsic.h @@ -186,11 +186,18 @@ enum HWIntrinsicFlag : unsigned int HW_Flag_ReturnsPerElementMask = 0x10000, // The intrinsic uses a mask in arg1 to select elements present in the result - HW_Flag_MaskedOperation = 0x20000, + HW_Flag_ExplicitMaskedOperation = 0x20000, // The intrinsic uses a mask in arg1 to select elements present in the result, and must use a low register. HW_Flag_LowMaskedOperation = 0x40000, + // The intrinsic can optionally use a mask in arg1 to select elements present in the result, which is not present in + // the API call + HW_Flag_OptionalEmbeddedMaskedOperation = 0x80000, + + // The intrinsic uses a mask in arg1 to select elements present in the result, which is not present in the API call + HW_Flag_EmbeddedMaskedOperation = 0x100000, + #else #error Unsupported platform #endif @@ -872,7 +879,7 @@ struct HWIntrinsicInfo static bool IsMaskedOperation(NamedIntrinsic id) { const HWIntrinsicFlag flags = lookupFlags(id); - return ((flags & HW_Flag_MaskedOperation) != 0) || IsLowMaskedOperation(id); + return IsLowMaskedOperation(id) || IsOptionalEmbeddedMaskedOperation(id) || IsExplicitMaskedOperation(id); } static bool IsLowMaskedOperation(NamedIntrinsic id) @@ -881,6 +888,24 @@ struct HWIntrinsicInfo return (flags & HW_Flag_LowMaskedOperation) != 0; } + static bool IsOptionalEmbeddedMaskedOperation(NamedIntrinsic id) + { + const HWIntrinsicFlag flags = lookupFlags(id); + return (flags & HW_Flag_OptionalEmbeddedMaskedOperation) != 0; + } + + static bool IsEmbeddedMaskedOperation(NamedIntrinsic id) + { + const HWIntrinsicFlag flags = lookupFlags(id); + return (flags & HW_Flag_EmbeddedMaskedOperation) != 0; + } + + static bool IsExplicitMaskedOperation(NamedIntrinsic id) + { + const HWIntrinsicFlag flags = lookupFlags(id); + return (flags & HW_Flag_ExplicitMaskedOperation) != 0; + } + #endif // TARGET_ARM64 static bool HasSpecialSideEffect(NamedIntrinsic id) diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index 8e3288f75d709..205102abc9b0b 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -2222,9 +2222,8 @@ GenTree* Compiler::gtNewSimdConvertVectorToMaskNode(var_types type, assert(varTypeIsSIMD(node)); // ConvertVectorToMask uses cmpne which requires an embedded mask. - GenTree* embeddedMask = gtNewSimdHWIntrinsicNode(TYP_MASK, NI_Sve_CreateTrueMaskAll, simdBaseJitType, simdSize); - return gtNewSimdHWIntrinsicNode(TYP_MASK, embeddedMask, node, NI_Sve_ConvertVectorToMask, simdBaseJitType, - simdSize); + GenTree* trueMask = gtNewSimdAllTrueMaskNode(simdBaseJitType, simdSize); + return gtNewSimdHWIntrinsicNode(TYP_MASK, trueMask, node, NI_Sve_ConvertVectorToMask, simdBaseJitType, simdSize); } //------------------------------------------------------------------------ @@ -2246,4 +2245,19 @@ GenTree* Compiler::gtNewSimdConvertMaskToVectorNode(GenTreeHWIntrinsic* node, va node->GetSimdSize()); } +//------------------------------------------------------------------------ +// gtNewSimdEmbeddedMaskNode: Create an embedded mask +// +// Arguments: +// simdBaseJitType -- the base jit type of the nodes being masked +// simdSize -- the simd size of the nodes being masked +// +// Return Value: +// The mask +// +GenTree* Compiler::gtNewSimdAllTrueMaskNode(CorInfoType simdBaseJitType, unsigned simdSize) +{ + return gtNewSimdHWIntrinsicNode(TYP_MASK, NI_Sve_CreateTrueMaskAll, simdBaseJitType, simdSize); +} + #endif // FEATURE_HW_INTRINSICS diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 9a3a98e087a27..c5ea702c1c4be 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -401,6 +401,100 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) unreached(); } } + else if (intrin.numOperands >= 2 && intrin.op2->IsEmbMaskOp()) + { + // Handle case where op2 is operation that needs embedded mask + GenTree* op2 = intrin.op2; + assert(intrin.id == NI_Sve_ConditionalSelect); + assert(op2->isContained()); + assert(op2->OperIsHWIntrinsic()); + + // Get the registers and intrinsics that needs embedded mask + const HWIntrinsic intrinEmbMask(op2->AsHWIntrinsic()); + instruction insEmbMask = HWIntrinsicInfo::lookupIns(intrinEmbMask.id, intrinEmbMask.baseType); + const bool instrIsRMW = op2->isRMWHWIntrinsic(compiler); + + regNumber maskReg = op1Reg; + regNumber embMaskOp1Reg = REG_NA; + regNumber embMaskOp2Reg = REG_NA; + regNumber falseReg = op3Reg; + + switch (intrinEmbMask.numOperands) + { + case 2: + assert(intrinEmbMask.op2 != nullptr); + embMaskOp2Reg = intrinEmbMask.op2->GetRegNum(); + FALLTHROUGH; + + case 1: + assert(intrinEmbMask.op1 != nullptr); + embMaskOp1Reg = intrinEmbMask.op1->GetRegNum(); + break; + + default: + unreached(); + } + + switch (intrinEmbMask.numOperands) + { + case 1: + assert(!instrIsRMW); + if (targetReg != falseReg) + { + GetEmitter()->emitIns_R_R(INS_sve_movprfx, EA_SCALABLE, targetReg, falseReg); + } + GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, opt); + break; + + case 2: + + assert(instrIsRMW); + + if (intrin.op3->IsVectorZero()) + { + // If `falseReg` is zero, then move the first operand of `intrinEmbMask` in the + // destination using /Z. + GetEmitter()->emitIns_R_R_R(INS_sve_movprfx, emitSize, targetReg, maskReg, embMaskOp1Reg, opt); + + // Finally, perform the actual "predicated" operation so that `targetReg` is the first operand + // and `embMaskOp2Reg` is the second operand. + GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp2Reg, opt); + } + else if (targetReg != falseReg) + { + // If `targetReg` and `falseReg` are not same, then we need to move it to `targetReg` first + // so the `insEmbMask` operation can be merged on top of it. + + if (falseReg != embMaskOp1Reg) + { + // None of targetReg, embMaskOp1Reg and falseReg are same. In such case, use the + // "unpredicated" version of the instruction and then use "sel" to select the active lanes. + + GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, embMaskOp1Reg, embMaskOp2Reg, + opt, INS_SCALABLE_OPTS_UNPREDICATED); + GetEmitter()->emitIns_R_R_R_R(INS_sve_sel, emitSize, targetReg, maskReg, targetReg, + falseReg, opt, INS_SCALABLE_OPTS_UNPREDICATED); + break; + } + else if (targetReg != embMaskOp1Reg) + { + // embMaskOp1Reg is same as `falseReg`, but not same as `targetReg`. Move the + // `embMaskOp1Reg` i.e. `falseReg` in `targetReg`, using "unpredicated movprfx", so the + // subsequent `insEmbMask` operation can be merged on top of it. + GetEmitter()->emitIns_R_R(INS_sve_movprfx, EA_SCALABLE, targetReg, falseReg, opt); + } + + // Finally, perform the actual "predicated" operation so that `targetReg` is the first operand + // and `embMaskOp2Reg` is the second operand. + GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp2Reg, opt); + } + + break; + + default: + unreached(); + } + } else { assert(!hasImmediateOperand); @@ -419,6 +513,14 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { GetEmitter()->emitIns_R_R(ins, emitSize, targetReg, op1Reg, opt); } + else if (HWIntrinsicInfo::IsScalable(intrin.id)) + { + assert(!node->IsEmbMaskOp()); + // This generates an unpredicated version + // Predicated should be taken care above `intrin.op2->IsEmbMaskOp()` + GetEmitter()->emitIns_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, opt, + INS_SCALABLE_OPTS_UNPREDICATED); + } else if (isRMW) { if (targetReg != op1Reg) @@ -437,17 +539,24 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) break; case 3: - assert(isRMW); - if (targetReg != op1Reg) + if (isRMW) { - assert(targetReg != op2Reg); - assert(targetReg != op3Reg); + if (targetReg != op1Reg) + { + assert(targetReg != op2Reg); + assert(targetReg != op3Reg); - GetEmitter()->emitIns_Mov(INS_mov, emitTypeSize(node), targetReg, op1Reg, /* canSkip */ true); + GetEmitter()->emitIns_Mov(INS_mov, emitTypeSize(node), targetReg, op1Reg, + /* canSkip */ true); + } + GetEmitter()->emitIns_R_R_R(ins, emitSize, targetReg, op2Reg, op3Reg, opt); + } + else + { + GetEmitter()->emitIns_R_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, op3Reg, opt, + INS_SCALABLE_OPTS_UNPREDICATED); } - GetEmitter()->emitIns_R_R_R(ins, emitSize, targetReg, op2Reg, op3Reg, opt); break; - default: unreached(); } diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index ac110c2a0e1b5..14b880c8e570e 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -17,6 +17,9 @@ // SVE Intrinsics // Sve +HARDWARE_INTRINSIC(Sve, Abs, -1, -1, false, {INS_sve_abs, INS_invalid, INS_sve_abs, INS_invalid, INS_sve_abs, INS_invalid, INS_sve_abs, INS_invalid, INS_sve_fabs, INS_sve_fabs}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation) +HARDWARE_INTRINSIC(Sve, Add, -1, -1, false, {INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_fadd, INS_sve_fadd}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ConditionalSelect, -1, 3, true, {INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_SupportsContainment) HARDWARE_INTRINSIC(Sve, CreateTrueMaskByte, -1, 1, false, {INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_EnumPattern, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, CreateTrueMaskDouble, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue}, HW_Category_EnumPattern, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, CreateTrueMaskInt16, -1, 1, false, {INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_EnumPattern, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_ReturnsPerElementMask) @@ -28,7 +31,7 @@ HARDWARE_INTRINSIC(Sve, CreateTrueMaskUInt16, HARDWARE_INTRINSIC(Sve, CreateTrueMaskUInt32, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_EnumPattern, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, CreateTrueMaskUInt64, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid}, HW_Category_EnumPattern, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, LoadVector, -1, 2, true, {INS_sve_ld1b, INS_sve_ld1b, INS_sve_ld1h, INS_sve_ld1h, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1d, INS_sve_ld1d, INS_sve_ld1w, INS_sve_ld1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVector, -1, 2, true, {INS_sve_ld1b, INS_sve_ld1b, INS_sve_ld1h, INS_sve_ld1h, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1d, INS_sve_ld1d, INS_sve_ld1w, INS_sve_ld1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) @@ -38,11 +41,11 @@ HARDWARE_INTRINSIC(Sve, LoadVector, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Special intrinsics that are generated during importing or lowering -HARDWARE_INTRINSIC(Sve, ConvertMaskToVector, -1, 1, true, {INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov}, HW_Category_Helper, HW_Flag_Scalable|HW_Flag_MaskedOperation) +HARDWARE_INTRINSIC(Sve, ConvertMaskToVector, -1, 1, true, {INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov}, HW_Category_Helper, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation) HARDWARE_INTRINSIC(Sve, ConvertVectorToMask, -1, 2, true, {INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne}, HW_Category_Helper, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_LowMaskedOperation) - HARDWARE_INTRINSIC(Sve, CreateTrueMaskAll, -1, -1, false, {INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue}, HW_Category_Helper, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) + #endif // FEATURE_HW_INTRINSIC #undef HARDWARE_INTRINSIC diff --git a/src/coreclr/jit/liveness.cpp b/src/coreclr/jit/liveness.cpp index 521e989133433..ef3fedf5b31d3 100644 --- a/src/coreclr/jit/liveness.cpp +++ b/src/coreclr/jit/liveness.cpp @@ -1762,8 +1762,8 @@ void Compiler::fgComputeLifeLIR(VARSET_TP& life, BasicBlock* block, VARSET_VALAR operand->SetUnusedValue(); } - // Special-case PUTARG_STK: since this operator is not considered a value, DCE will not remove - // these nodes. + // Special-case PUTARG_STK: since this operator is not considered a value, DCE will not + // remove these nodes. if (operand->OperIs(GT_PUTARG_STK)) { operand->AsPutArgStk()->gtOp1->SetUnusedValue(); diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index cbdc886ee2802..7ea9a90d62343 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -1280,6 +1280,34 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) break; } + if (HWIntrinsicInfo::IsEmbeddedMaskedOperation(intrinsicId)) + { + LIR::Use use; + if (BlockRange().TryGetUse(node, &use)) + { + GenTree* user = use.User(); + // Wrap the intrinsic in ConditionalSelect only if it is not already inside another ConditionalSelect + if (!user->OperIsHWIntrinsic() || (user->AsHWIntrinsic()->GetHWIntrinsicId() != NI_Sve_ConditionalSelect)) + { + CorInfoType simdBaseJitType = node->GetSimdBaseJitType(); + unsigned simdSize = node->GetSimdSize(); + var_types simdType = Compiler::getSIMDTypeForSize(simdSize); + GenTree* trueMask = comp->gtNewSimdAllTrueMaskNode(simdBaseJitType, simdSize); + GenTree* trueVal = node; + GenTree* falseVal = comp->gtNewZeroConNode(simdType); + + GenTreeHWIntrinsic* condSelNode = + comp->gtNewSimdHWIntrinsicNode(simdType, trueMask, trueVal, falseVal, NI_Sve_ConditionalSelect, + simdBaseJitType, simdSize); + + BlockRange().InsertBefore(node, trueMask); + BlockRange().InsertBefore(node, falseVal); + BlockRange().InsertAfter(node, condSelNode); + use.ReplaceWith(condSelNode); + } + } + } + ContainCheckHWIntrinsic(node); return node->gtNext; } @@ -3275,6 +3303,45 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) } break; + case NI_Sve_ConditionalSelect: + { + assert(intrin.numOperands == 3); + GenTree* op1 = intrin.op1; + GenTree* op2 = intrin.op2; + GenTree* op3 = intrin.op3; + + // Handle op1 + if (op1->IsVectorZero()) + { + // When we are merging with zero, we can specialize + // and avoid instantiating the vector constant. + MakeSrcContained(node, op1); + } + + // Handle op2 + if (op2->OperIsHWIntrinsic()) + { + uint32_t maskSize = genTypeSize(node->GetSimdBaseType()); + uint32_t operSize = genTypeSize(op2->AsHWIntrinsic()->GetSimdBaseType()); + + if ((maskSize == operSize) && IsInvariantInRange(op2, node)) + { + MakeSrcContained(node, op2); + op2->MakeEmbMaskOp(); + } + } + + // Handle op3 + if (op3->IsVectorZero()) + { + // When we are merging with zero, we can specialize + // and avoid instantiating the vector constant. + MakeSrcContained(node, op3); + } + + break; + } + default: unreached(); } diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index 5a9b12ca4aa27..0e18c5685066f 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -2970,7 +2970,7 @@ GenTree* Lowering::LowerHWIntrinsicCndSel(GenTreeHWIntrinsic* node) } //---------------------------------------------------------------------------------------------- -// Lowering::LowerHWIntrinsicCndSel: Lowers an AVX512 TernaryLogic call +// Lowering::LowerHWIntrinsicTernaryLogic: Lowers an AVX512 TernaryLogic call // // Arguments: // node - The hardware intrinsic node. @@ -10137,7 +10137,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) // contained and not a memory operand and know to invoke the special handling // so that the embedded masking can work as expected. - if (op2->isEvexEmbeddedMaskingCompatibleHWIntrinsic()) + if (op2->isEmbeddedMaskingCompatibleHWIntrinsic()) { uint32_t maskSize = genTypeSize(simdBaseType); uint32_t operSize = genTypeSize(op2->AsHWIntrinsic()->GetSimdBaseType()); diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 3ee0f88ee2214..37fa4320cc691 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -3489,6 +3489,16 @@ int LinearScan::BuildOperandUses(GenTree* node, regMaskTP candidates) if (numArgs != 1) { +#ifdef TARGET_ARM64 + if (HWIntrinsicInfo::IsScalable(hwintrinsic->GetHWIntrinsicId())) + { + for (size_t argNum = 1; argNum <= numArgs; argNum++) + { + BuildOperandUses(hwintrinsic->Op(argNum), candidates); + } + return (int)numArgs; + } +#endif assert(numArgs == 2); assert(hwintrinsic->Op(2)->isContained()); assert(hwintrinsic->Op(2)->IsCnsIntOrI()); diff --git a/src/coreclr/jit/ssabuilder.cpp b/src/coreclr/jit/ssabuilder.cpp index caba13f3536ab..0a5229e32b8b7 100644 --- a/src/coreclr/jit/ssabuilder.cpp +++ b/src/coreclr/jit/ssabuilder.cpp @@ -965,8 +965,8 @@ void SsaBuilder::AddPhiArgsToSuccessors(BasicBlock* block) // Walk the statements for phi nodes. for (Statement* const stmt : succ->Statements()) { - // A prefix of the statements of the block are phi definition nodes. If we complete processing - // that prefix, exit. + // A prefix of the statements of the block are phi definition nodes. If we complete + // processing that prefix, exit. if (!stmt->IsPhiDefnStmt()) { break; @@ -988,8 +988,9 @@ void SsaBuilder::AddPhiArgsToSuccessors(BasicBlock* block) { if ((memoryKind == GcHeap) && m_pCompiler->byrefStatesMatchGcHeapStates) { - // We've already propagated the "out" number to the phi shared with ByrefExposed, - // but still need to update bbMemorySsaPhiFunc to be in sync between GcHeap and ByrefExposed. + // We've already propagated the "out" number to the phi shared with + // ByrefExposed, but still need to update bbMemorySsaPhiFunc to be in sync + // between GcHeap and ByrefExposed. assert(memoryKind > ByrefExposed); assert(block->bbMemorySsaNumOut[memoryKind] == block->bbMemorySsaNumOut[ByrefExposed]); assert((succ->bbMemorySsaPhiFunc[ByrefExposed] == succMemoryPhi) || @@ -1009,8 +1010,9 @@ void SsaBuilder::AddPhiArgsToSuccessors(BasicBlock* block) BasicBlock::MemoryPhiArg* curArg = succMemoryPhi; unsigned ssaNum = block->bbMemorySsaNumOut[memoryKind]; bool found = false; - // This is a quadratic algorithm. We might need to consider some switch over to a hash table - // representation for the arguments of a phi node, to make this linear. + // This is a quadratic algorithm. We might need to consider some switch over + // to a hash table representation for the arguments of a phi node, to make this + // linear. while (curArg != nullptr) { if (curArg->m_ssaNum == ssaNum) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs index fbd5ee65ca748..8d56e32fc863c 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs @@ -31,6 +31,125 @@ internal Arm64() { } public static new bool IsSupported { [Intrinsic] get { return false; } } } + + /// Abs : Absolute value + + /// + /// svint8_t svabs[_s8]_m(svint8_t inactive, svbool_t pg, svint8_t op) + /// svint8_t svabs[_s8]_x(svbool_t pg, svint8_t op) + /// svint8_t svabs[_s8]_z(svbool_t pg, svint8_t op) + /// + public static unsafe Vector Abs(Vector value) { throw new PlatformNotSupportedException(); } + + /// + /// svint16_t svabs[_s16]_m(svint16_t inactive, svbool_t pg, svint16_t op) + /// svint16_t svabs[_s16]_x(svbool_t pg, svint16_t op) + /// svint16_t svabs[_s16]_z(svbool_t pg, svint16_t op) + /// + public static unsafe Vector Abs(Vector value) { throw new PlatformNotSupportedException(); } + + /// + /// svint32_t svabs[_s32]_m(svint32_t inactive, svbool_t pg, svint32_t op) + /// svint32_t svabs[_s32]_x(svbool_t pg, svint32_t op) + /// svint32_t svabs[_s32]_z(svbool_t pg, svint32_t op) + /// + public static unsafe Vector Abs(Vector value) { throw new PlatformNotSupportedException(); } + + /// + /// svint64_t svabs[_s64]_m(svint64_t inactive, svbool_t pg, svint64_t op) + /// svint64_t svabs[_s64]_x(svbool_t pg, svint64_t op) + /// svint64_t svabs[_s64]_z(svbool_t pg, svint64_t op) + /// + public static unsafe Vector Abs(Vector value) { throw new PlatformNotSupportedException(); } + + /// + /// svfloat32_t svabs[_f32]_m(svfloat32_t inactive, svbool_t pg, svfloat32_t op) + /// svfloat32_t svabs[_f32]_x(svbool_t pg, svfloat32_t op) + /// svfloat32_t svabs[_f32]_z(svbool_t pg, svfloat32_t op) + /// + public static unsafe Vector Abs(Vector value) { throw new PlatformNotSupportedException(); } + + /// + /// svfloat64_t svabs[_f64]_m(svfloat64_t inactive, svbool_t pg, svfloat64_t op) + /// svfloat64_t svabs[_f64]_x(svbool_t pg, svfloat64_t op) + /// svfloat64_t svabs[_f64]_z(svbool_t pg, svfloat64_t op) + /// + public static unsafe Vector Abs(Vector value) { throw new PlatformNotSupportedException(); } + + + /// Add : Add + + /// + /// svint8_t svadd[_s8]_m(svbool_t pg, svint8_t op1, svint8_t op2) + /// svint8_t svadd[_s8]_x(svbool_t pg, svint8_t op1, svint8_t op2) + /// svint8_t svadd[_s8]_z(svbool_t pg, svint8_t op1, svint8_t op2) + /// + public static unsafe Vector Add(Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svint16_t svadd[_s16]_m(svbool_t pg, svint16_t op1, svint16_t op2) + /// svint16_t svadd[_s16]_x(svbool_t pg, svint16_t op1, svint16_t op2) + /// svint16_t svadd[_s16]_z(svbool_t pg, svint16_t op1, svint16_t op2) + /// + public static unsafe Vector Add(Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svint32_t svadd[_s32]_m(svbool_t pg, svint32_t op1, svint32_t op2) + /// svint32_t svadd[_s32]_x(svbool_t pg, svint32_t op1, svint32_t op2) + /// svint32_t svadd[_s32]_z(svbool_t pg, svint32_t op1, svint32_t op2) + /// + public static unsafe Vector Add(Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svint64_t svadd[_s64]_m(svbool_t pg, svint64_t op1, svint64_t op2) + /// svint64_t svadd[_s64]_x(svbool_t pg, svint64_t op1, svint64_t op2) + /// svint64_t svadd[_s64]_z(svbool_t pg, svint64_t op1, svint64_t op2) + /// + public static unsafe Vector Add(Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svuint8_t svadd[_u8]_m(svbool_t pg, svuint8_t op1, svuint8_t op2) + /// svuint8_t svadd[_u8]_x(svbool_t pg, svuint8_t op1, svuint8_t op2) + /// svuint8_t svadd[_u8]_z(svbool_t pg, svuint8_t op1, svuint8_t op2) + /// + public static unsafe Vector Add(Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svuint16_t svadd[_u16]_m(svbool_t pg, svuint16_t op1, svuint16_t op2) + /// svuint16_t svadd[_u16]_x(svbool_t pg, svuint16_t op1, svuint16_t op2) + /// svuint16_t svadd[_u16]_z(svbool_t pg, svuint16_t op1, svuint16_t op2) + /// + public static unsafe Vector Add(Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svuint32_t svadd[_u32]_m(svbool_t pg, svuint32_t op1, svuint32_t op2) + /// svuint32_t svadd[_u32]_x(svbool_t pg, svuint32_t op1, svuint32_t op2) + /// svuint32_t svadd[_u32]_z(svbool_t pg, svuint32_t op1, svuint32_t op2) + /// + public static unsafe Vector Add(Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svuint64_t svadd[_u64]_m(svbool_t pg, svuint64_t op1, svuint64_t op2) + /// svuint64_t svadd[_u64]_x(svbool_t pg, svuint64_t op1, svuint64_t op2) + /// svuint64_t svadd[_u64]_z(svbool_t pg, svuint64_t op1, svuint64_t op2) + /// + public static unsafe Vector Add(Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svfloat32_t svadd[_f32]_m(svbool_t pg, svfloat32_t op1, svfloat32_t op2) + /// svfloat32_t svadd[_f32]_x(svbool_t pg, svfloat32_t op1, svfloat32_t op2) + /// svfloat32_t svadd[_f32]_z(svbool_t pg, svfloat32_t op1, svfloat32_t op2) + /// + public static unsafe Vector Add(Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svfloat64_t svadd[_f64]_m(svbool_t pg, svfloat64_t op1, svfloat64_t op2) + /// svfloat64_t svadd[_f64]_x(svbool_t pg, svfloat64_t op1, svfloat64_t op2) + /// svfloat64_t svadd[_f64]_z(svbool_t pg, svfloat64_t op1, svfloat64_t op2) + /// + public static unsafe Vector Add(Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// CreateTrueMaskByte : Set predicate elements to true /// @@ -120,7 +239,65 @@ internal Arm64() { } /// public static unsafe Vector CreateTrueMaskUInt64([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw new PlatformNotSupportedException(); } + /// ConditionalSelect : Conditionally select elements + + /// + /// svint8_t svsel[_s8](svbool_t pg, svint8_t op1, svint8_t op2) + /// svbool_t svsel[_b](svbool_t pg, svbool_t op1, svbool_t op2) + /// + public static unsafe Vector ConditionalSelect(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svint16_t svsel[_s16](svbool_t pg, svint16_t op1, svint16_t op2) + /// svbool_t svsel[_b](svbool_t pg, svbool_t op1, svbool_t op2) + /// + public static unsafe Vector ConditionalSelect(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svint32_t svsel[_s32](svbool_t pg, svint32_t op1, svint32_t op2) + /// svbool_t svsel[_b](svbool_t pg, svbool_t op1, svbool_t op2) + /// + public static unsafe Vector ConditionalSelect(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + /// + /// svint64_t svsel[_s64](svbool_t pg, svint64_t op1, svint64_t op2) + /// svbool_t svsel[_b](svbool_t pg, svbool_t op1, svbool_t op2) + /// + public static unsafe Vector ConditionalSelect(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svuint8_t svsel[_u8](svbool_t pg, svuint8_t op1, svuint8_t op2) + /// svbool_t svsel[_b](svbool_t pg, svbool_t op1, svbool_t op2) + /// + public static unsafe Vector ConditionalSelect(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svuint16_t svsel[_u16](svbool_t pg, svuint16_t op1, svuint16_t op2) + /// svbool_t svsel[_b](svbool_t pg, svbool_t op1, svbool_t op2) + /// + public static unsafe Vector ConditionalSelect(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svuint32_t svsel[_u32](svbool_t pg, svuint32_t op1, svuint32_t op2) + /// svbool_t svsel[_b](svbool_t pg, svbool_t op1, svbool_t op2) + /// + public static unsafe Vector ConditionalSelect(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svuint64_t svsel[_u64](svbool_t pg, svuint64_t op1, svuint64_t op2) + /// svbool_t svsel[_b](svbool_t pg, svbool_t op1, svbool_t op2) + /// + public static unsafe Vector ConditionalSelect(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svfloat32_t svsel[_f32](svbool_t pg, svfloat32_t op1, svfloat32_t op2) + /// + public static unsafe Vector ConditionalSelect(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svfloat64_t svsel[_f64](svbool_t pg, svfloat64_t op1, svfloat64_t op2) + /// + public static unsafe Vector ConditionalSelect(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } /// LoadVector : Unextended load diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs index 6ba2a2c67bc8a..a13957916f49a 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs @@ -29,6 +29,154 @@ internal Arm64() { } } + /// Abs : Absolute value + + /// + /// svint8_t svabs[_s8]_m(svint8_t inactive, svbool_t pg, svint8_t op) + /// ABS Ztied.B, Pg/M, Zop.B + /// MOVPRFX Zresult, Zinactive; ABS Zresult.B, Pg/M, Zop.B + /// svint8_t svabs[_s8]_x(svbool_t pg, svint8_t op) + /// ABS Ztied.B, Pg/M, Ztied.B + /// MOVPRFX Zresult, Zop; ABS Zresult.B, Pg/M, Zop.B + /// svint8_t svabs[_s8]_z(svbool_t pg, svint8_t op) + /// MOVPRFX Zresult.B, Pg/Z, Zop.B; ABS Zresult.B, Pg/M, Zop.B + /// + public static unsafe Vector Abs(Vector value) => Abs(value); + + /// + /// svint16_t svabs[_s16]_m(svint16_t inactive, svbool_t pg, svint16_t op) + /// ABS Ztied.H, Pg/M, Zop.H + /// MOVPRFX Zresult, Zinactive; ABS Zresult.H, Pg/M, Zop.H + /// svint16_t svabs[_s16]_x(svbool_t pg, svint16_t op) + /// ABS Ztied.H, Pg/M, Ztied.H + /// MOVPRFX Zresult, Zop; ABS Zresult.H, Pg/M, Zop.H + /// svint16_t svabs[_s16]_z(svbool_t pg, svint16_t op) + /// MOVPRFX Zresult.H, Pg/Z, Zop.H; ABS Zresult.H, Pg/M, Zop.H + /// + public static unsafe Vector Abs(Vector value) => Abs(value); + + /// + /// svint32_t svabs[_s32]_m(svint32_t inactive, svbool_t pg, svint32_t op) + /// ABS Ztied.S, Pg/M, Zop.S + /// MOVPRFX Zresult, Zinactive; ABS Zresult.S, Pg/M, Zop.S + /// svint32_t svabs[_s32]_x(svbool_t pg, svint32_t op) + /// ABS Ztied.S, Pg/M, Ztied.S + /// MOVPRFX Zresult, Zop; ABS Zresult.S, Pg/M, Zop.S + /// svint32_t svabs[_s32]_z(svbool_t pg, svint32_t op) + /// MOVPRFX Zresult.S, Pg/Z, Zop.S; ABS Zresult.S, Pg/M, Zop.S + /// + public static unsafe Vector Abs(Vector value) => Abs(value); + + /// + /// svint64_t svabs[_s64]_m(svint64_t inactive, svbool_t pg, svint64_t op) + /// ABS Ztied.D, Pg/M, Zop.D + /// MOVPRFX Zresult, Zinactive; ABS Zresult.D, Pg/M, Zop.D + /// svint64_t svabs[_s64]_x(svbool_t pg, svint64_t op) + /// ABS Ztied.D, Pg/M, Ztied.D + /// MOVPRFX Zresult, Zop; ABS Zresult.D, Pg/M, Zop.D + /// svint64_t svabs[_s64]_z(svbool_t pg, svint64_t op) + /// MOVPRFX Zresult.D, Pg/Z, Zop.D; ABS Zresult.D, Pg/M, Zop.D + /// + public static unsafe Vector Abs(Vector value) => Abs(value); + + /// + /// svfloat32_t svabs[_f32]_m(svfloat32_t inactive, svbool_t pg, svfloat32_t op) + /// FABS Ztied.S, Pg/M, Zop.S + /// MOVPRFX Zresult, Zinactive; FABS Zresult.S, Pg/M, Zop.S + /// svfloat32_t svabs[_f32]_x(svbool_t pg, svfloat32_t op) + /// FABS Ztied.S, Pg/M, Ztied.S + /// MOVPRFX Zresult, Zop; FABS Zresult.S, Pg/M, Zop.S + /// svfloat32_t svabs[_f32]_z(svbool_t pg, svfloat32_t op) + /// MOVPRFX Zresult.S, Pg/Z, Zop.S; FABS Zresult.S, Pg/M, Zop.S + /// + public static unsafe Vector Abs(Vector value) => Abs(value); + + /// + /// svfloat64_t svabs[_f64]_m(svfloat64_t inactive, svbool_t pg, svfloat64_t op) + /// FABS Ztied.D, Pg/M, Zop.D + /// MOVPRFX Zresult, Zinactive; FABS Zresult.D, Pg/M, Zop.D + /// svfloat64_t svabs[_f64]_x(svbool_t pg, svfloat64_t op) + /// FABS Ztied.D, Pg/M, Ztied.D + /// MOVPRFX Zresult, Zop; FABS Zresult.D, Pg/M, Zop.D + /// svfloat64_t svabs[_f64]_z(svbool_t pg, svfloat64_t op) + /// MOVPRFX Zresult.D, Pg/Z, Zop.D; FABS Zresult.D, Pg/M, Zop.D + /// + public static unsafe Vector Abs(Vector value) => Abs(value); + + + /// Add : Add + + /// + /// svint8_t svadd[_s8]_m(svbool_t pg, svint8_t op1, svint8_t op2) + /// svint8_t svadd[_s8]_x(svbool_t pg, svint8_t op1, svint8_t op2) + /// svint8_t svadd[_s8]_z(svbool_t pg, svint8_t op1, svint8_t op2) + /// + public static unsafe Vector Add(Vector left, Vector right) => Add(left, right); + + /// + /// svint16_t svadd[_s16]_m(svbool_t pg, svint16_t op1, svint16_t op2) + /// svint16_t svadd[_s16]_x(svbool_t pg, svint16_t op1, svint16_t op2) + /// svint16_t svadd[_s16]_z(svbool_t pg, svint16_t op1, svint16_t op2) + /// + public static unsafe Vector Add(Vector left, Vector right) => Add(left, right); + + /// + /// svint32_t svadd[_s32]_m(svbool_t pg, svint32_t op1, svint32_t op2) + /// svint32_t svadd[_s32]_x(svbool_t pg, svint32_t op1, svint32_t op2) + /// svint32_t svadd[_s32]_z(svbool_t pg, svint32_t op1, svint32_t op2) + /// + public static unsafe Vector Add(Vector left, Vector right) => Add(left, right); + + /// + /// svint64_t svadd[_s64]_m(svbool_t pg, svint64_t op1, svint64_t op2) + /// svint64_t svadd[_s64]_x(svbool_t pg, svint64_t op1, svint64_t op2) + /// svint64_t svadd[_s64]_z(svbool_t pg, svint64_t op1, svint64_t op2) + /// + public static unsafe Vector Add(Vector left, Vector right) => Add(left, right); + + /// + /// svuint8_t svadd[_u8]_m(svbool_t pg, svuint8_t op1, svuint8_t op2) + /// svuint8_t svadd[_u8]_x(svbool_t pg, svuint8_t op1, svuint8_t op2) + /// svuint8_t svadd[_u8]_z(svbool_t pg, svuint8_t op1, svuint8_t op2) + /// + public static unsafe Vector Add(Vector left, Vector right) => Add(left, right); + + /// + /// svuint16_t svadd[_u16]_m(svbool_t pg, svuint16_t op1, svuint16_t op2) + /// svuint16_t svadd[_u16]_x(svbool_t pg, svuint16_t op1, svuint16_t op2) + /// svuint16_t svadd[_u16]_z(svbool_t pg, svuint16_t op1, svuint16_t op2) + /// + public static unsafe Vector Add(Vector left, Vector right) => Add(left, right); + + /// + /// svuint32_t svadd[_u32]_m(svbool_t pg, svuint32_t op1, svuint32_t op2) + /// svuint32_t svadd[_u32]_x(svbool_t pg, svuint32_t op1, svuint32_t op2) + /// svuint32_t svadd[_u32]_z(svbool_t pg, svuint32_t op1, svuint32_t op2) + /// + public static unsafe Vector Add(Vector left, Vector right) => Add(left, right); + + /// + /// svuint64_t svadd[_u64]_m(svbool_t pg, svuint64_t op1, svuint64_t op2) + /// svuint64_t svadd[_u64]_x(svbool_t pg, svuint64_t op1, svuint64_t op2) + /// svuint64_t svadd[_u64]_z(svbool_t pg, svuint64_t op1, svuint64_t op2) + /// + public static unsafe Vector Add(Vector left, Vector right) => Add(left, right); + + /// + /// svfloat32_t svadd[_f32]_m(svbool_t pg, svfloat32_t op1, svfloat32_t op2) + /// svfloat32_t svadd[_f32]_x(svbool_t pg, svfloat32_t op1, svfloat32_t op2) + /// svfloat32_t svadd[_f32]_z(svbool_t pg, svfloat32_t op1, svfloat32_t op2) + /// + public static unsafe Vector Add(Vector left, Vector right) => Add(left, right); + + /// + /// svfloat64_t svadd[_f64]_m(svbool_t pg, svfloat64_t op1, svfloat64_t op2) + /// svfloat64_t svadd[_f64]_x(svbool_t pg, svfloat64_t op1, svfloat64_t op2) + /// svfloat64_t svadd[_f64]_z(svbool_t pg, svfloat64_t op1, svfloat64_t op2) + /// + public static unsafe Vector Add(Vector left, Vector right) => Add(left, right); + + /// CreateTrueMaskByte : Set predicate elements to true /// @@ -118,7 +266,93 @@ internal Arm64() { } /// public static unsafe Vector CreateTrueMaskUInt64([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) => CreateTrueMaskUInt64(pattern); + /// ConditionalSelect : Conditionally select elements + + /// + /// svint8_t svsel[_s8](svbool_t pg, svint8_t op1, svint8_t op2) + /// SEL Zresult.B, Pg, Zop1.B, Zop2.B + /// svbool_t svsel[_b](svbool_t pg, svbool_t op1, svbool_t op2) + /// SEL Presult.B, Pg, Pop1.B, Pop2.B + /// + /// + public static unsafe Vector ConditionalSelect(Vector mask, Vector left, Vector right) => ConditionalSelect(mask, left, right); + + /// + /// svint16_t svsel[_s16](svbool_t pg, svint16_t op1, svint16_t op2) + /// SEL Zresult.H, Pg, Zop1.H, Zop2.H + /// svbool_t svsel[_b](svbool_t pg, svbool_t op1, svbool_t op2) + /// SEL Presult.B, Pg, Pop1.B, Pop2.B + /// + /// + public static unsafe Vector ConditionalSelect(Vector mask, Vector left, Vector right) => ConditionalSelect(mask, left, right); + + /// + /// svint32_t svsel[_s32](svbool_t pg, svint32_t op1, svint32_t op2) + /// SEL Zresult.S, Pg, Zop1.S, Zop2.S + /// svbool_t svsel[_b](svbool_t pg, svbool_t op1, svbool_t op2) + /// SEL Presult.B, Pg, Pop1.B, Pop2.B + /// + /// + public static unsafe Vector ConditionalSelect(Vector mask, Vector left, Vector right) => ConditionalSelect(mask, left, right); + + /// + /// svint64_t svsel[_s64](svbool_t pg, svint64_t op1, svint64_t op2) + /// SEL Zresult.D, Pg, Zop1.D, Zop2.D + /// svbool_t svsel[_b](svbool_t pg, svbool_t op1, svbool_t op2) + /// SEL Presult.B, Pg, Pop1.B, Pop2.B + /// + /// + public static unsafe Vector ConditionalSelect(Vector mask, Vector left, Vector right) => ConditionalSelect(mask, left, right); + + /// + /// svuint8_t svsel[_u8](svbool_t pg, svuint8_t op1, svuint8_t op2) + /// SEL Zresult.B, Pg, Zop1.B, Zop2.B + /// svbool_t svsel[_b](svbool_t pg, svbool_t op1, svbool_t op2) + /// SEL Presult.B, Pg, Pop1.B, Pop2.B + /// + /// + public static unsafe Vector ConditionalSelect(Vector mask, Vector left, Vector right) => ConditionalSelect(mask, left, right); + + /// + /// svuint16_t svsel[_u16](svbool_t pg, svuint16_t op1, svuint16_t op2) + /// SEL Zresult.H, Pg, Zop1.H, Zop2.H + /// svbool_t svsel[_b](svbool_t pg, svbool_t op1, svbool_t op2) + /// SEL Presult.B, Pg, Pop1.B, Pop2.B + /// + /// + public static unsafe Vector ConditionalSelect(Vector mask, Vector left, Vector right) => ConditionalSelect(mask, left, right); + + /// + /// svuint32_t svsel[_u32](svbool_t pg, svuint32_t op1, svuint32_t op2) + /// SEL Zresult.S, Pg, Zop1.S, Zop2.S + /// svbool_t svsel[_b](svbool_t pg, svbool_t op1, svbool_t op2) + /// SEL Presult.B, Pg, Pop1.B, Pop2.B + /// + /// + public static unsafe Vector ConditionalSelect(Vector mask, Vector left, Vector right) => ConditionalSelect(mask, left, right); + + /// + /// svuint64_t svsel[_u64](svbool_t pg, svuint64_t op1, svuint64_t op2) + /// SEL Zresult.D, Pg, Zop1.D, Zop2.D + /// svbool_t svsel[_b](svbool_t pg, svbool_t op1, svbool_t op2) + /// SEL Presult.B, Pg, Pop1.B, Pop2.B + /// + /// + public static unsafe Vector ConditionalSelect(Vector mask, Vector left, Vector right) => ConditionalSelect(mask, left, right); + /// + /// svfloat32_t svsel[_f32](svbool_t pg, svfloat32_t op1, svfloat32_t op2) + /// SEL Zresult.S, Pg, Zop1.S, Zop2.S + /// + /// + public static unsafe Vector ConditionalSelect(Vector mask, Vector left, Vector right) => ConditionalSelect(mask, left, right); + + /// + /// svfloat64_t svsel[_f64](svbool_t pg, svfloat64_t op1, svfloat64_t op2) + /// SEL Zresult.D, Pg, Zop1.D, Zop2.D + /// + /// + public static unsafe Vector ConditionalSelect(Vector mask, Vector left, Vector right) => ConditionalSelect(mask, left, right); /// LoadVector : Unextended load diff --git a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs index c34725ba0bae5..a1813dba44719 100644 --- a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs +++ b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs @@ -4147,7 +4147,22 @@ internal Sve() { } internal Arm64() { } public static new bool IsSupported { get { throw null; } } } - + public static System.Numerics.Vector Abs(System.Numerics.Vector value) { throw null; } + public static System.Numerics.Vector Abs(System.Numerics.Vector value) { throw null; } + public static System.Numerics.Vector Abs(System.Numerics.Vector value) { throw null; } + public static System.Numerics.Vector Abs(System.Numerics.Vector value) { throw null; } + public static System.Numerics.Vector Abs(System.Numerics.Vector value) { throw null; } + public static System.Numerics.Vector Abs(System.Numerics.Vector value) { throw null; } + public static System.Numerics.Vector Add(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Add(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Add(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Add(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Add(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Add(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Add(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Add(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Add(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Add(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } public static System.Numerics.Vector CreateTrueMaskByte([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; } public static System.Numerics.Vector CreateTrueMaskDouble([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; } public static System.Numerics.Vector CreateTrueMaskInt16([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; } @@ -4158,7 +4173,16 @@ internal Arm64() { } public static System.Numerics.Vector CreateTrueMaskUInt16([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; } public static System.Numerics.Vector CreateTrueMaskUInt32([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; } public static System.Numerics.Vector CreateTrueMaskUInt64([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; } - + public static System.Numerics.Vector ConditionalSelect(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector ConditionalSelect(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector ConditionalSelect(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector ConditionalSelect(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector ConditionalSelect(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector ConditionalSelect(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector ConditionalSelect(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector ConditionalSelect(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector ConditionalSelect(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector ConditionalSelect(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } public static unsafe System.Numerics.Vector LoadVector(System.Numerics.Vector mask, sbyte* address) { throw null; } public static unsafe System.Numerics.Vector LoadVector(System.Numerics.Vector mask, short* address) { throw null; } public static unsafe System.Numerics.Vector LoadVector(System.Numerics.Vector mask, int* address) { throw null; } @@ -4169,7 +4193,6 @@ internal Arm64() { } public static unsafe System.Numerics.Vector LoadVector(System.Numerics.Vector mask, ulong* address) { throw null; } public static unsafe System.Numerics.Vector LoadVector(System.Numerics.Vector mask, float* address) { throw null; } public static unsafe System.Numerics.Vector LoadVector(System.Numerics.Vector mask, double* address) { throw null; } - } public enum SveMaskPattern : byte diff --git a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs index b5f42595ca77e..a2ebacb1ce8b8 100644 --- a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs +++ b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs @@ -47,6 +47,16 @@ } }"; +const string SimpleVecOpTest_ValidationLogicForCndSel = @"for (var i = 0; i < RetElementCount; i++) + { + {Op1BaseType} iterResult = (mask[i] != 0) ? {GetIterResult} : falseVal[i]; + if (iterResult != result[i]) + { + succeeded = false; + break; + } + }"; + const string VecPairBinOpTest_ValidationLogic = @" int index = 0; int half = RetElementCount / 2; @@ -107,7 +117,9 @@ ("_TernaryOpTestTemplate.template", "SimpleTernOpTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleOpTest_ValidationLogic }), ("_UnaryOpTestTemplate.template", "SecureHashUnOpTest.template", new Dictionary { ["TemplateName"] = "SecureHash", ["TemplateValidationLogic"] = SecureHashOpTest_ValidationLogic }), ("_BinaryOpTestTemplate.template", "SecureHashBinOpTest.template", new Dictionary { ["TemplateName"] = "SecureHash", ["TemplateValidationLogic"] = SecureHashOpTest_ValidationLogic }), - ("_TernaryOpTestTemplate.template", "SecureHashTernOpTest.template", new Dictionary { ["TemplateName"] = "SecureHash", ["TemplateValidationLogic"] = SecureHashOpTest_ValidationLogic }) + ("_TernaryOpTestTemplate.template", "SecureHashTernOpTest.template", new Dictionary { ["TemplateName"] = "SecureHash", ["TemplateValidationLogic"] = SecureHashOpTest_ValidationLogic }), + ("_SveUnaryOpTestTemplate.template", "SveSimpleVecOpTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleVecOpTest_ValidationLogicForCndSel }), + ("_SveBinaryOpTestTemplate.template", "SveVecBinOpTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleVecOpTest_ValidationLogicForCndSel }), }; (string templateFileName, Dictionary templateData)[] AdvSimdInputs = new [] @@ -2878,16 +2890,45 @@ (string templateFileName, Dictionary templateData)[] SveInputs = new [] { - ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_float", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_double", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_sbyte", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_short", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_int", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_long", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int64", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_byte", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_ushort", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_uint", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_ulong", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveSimpleVecOpTest.template", new Dictionary { ["TestName"] = "Sve_Abs_float", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Abs", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "-TestLibrary.Generator.GetSingle()", ["ValidateIterResult"] = "Helpers.Abs(firstOp[i]) != result[i]", ["GetIterResult"] = "Helpers.Abs(leftOp[i])"}), + ("SveSimpleVecOpTest.template", new Dictionary { ["TestName"] = "Sve_Abs_double", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Abs", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "-TestLibrary.Generator.GetDouble()", ["ValidateIterResult"] = "Helpers.Abs(firstOp[i]) != result[i]", ["GetIterResult"] = "Helpers.Abs(leftOp[i])"}), + ("SveSimpleVecOpTest.template", new Dictionary { ["TestName"] = "Sve_Abs_sbyte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Abs", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "(sbyte)-TestLibrary.Generator.GetSByte()", ["ValidateIterResult"] = "Helpers.Abs(firstOp[i]) != result[i]", ["GetIterResult"] = "(sbyte)Helpers.Abs(leftOp[i])"}), + ("SveSimpleVecOpTest.template", new Dictionary { ["TestName"] = "Sve_Abs_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Abs", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "(short)-TestLibrary.Generator.GetInt16()", ["ValidateIterResult"] = "Helpers.Abs(firstOp[i]) != result[i]", ["GetIterResult"] = "(short)Helpers.Abs(leftOp[i])"}), + ("SveSimpleVecOpTest.template", new Dictionary { ["TestName"] = "Sve_Abs_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Abs", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "-TestLibrary.Generator.GetInt32()", ["ValidateIterResult"] = "Helpers.Abs(firstOp[i]) != result[i]", ["GetIterResult"] = "(int)Helpers.Abs(leftOp[i])"}), + ("SveSimpleVecOpTest.template", new Dictionary { ["TestName"] = "Sve_Abs_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Abs", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "-TestLibrary.Generator.GetInt64()", ["ValidateIterResult"] = "(long)Helpers.Abs(firstOp[i]) != (long)result[i]", ["GetIterResult"] = "(long)Helpers.Abs(leftOp[i])"}), + + ("SveVecBinOpTest.template", new Dictionary { ["TestName"] = "Sve_Add_float", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Add", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateIterResult"] = "Helpers.Add(left[i], right[i]) != result[i]", ["GetIterResult"] = "Helpers.Add(left[i], right[i])"}), + ("SveVecBinOpTest.template", new Dictionary { ["TestName"] = "Sve_Add_double", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Add", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateIterResult"] = "Helpers.Add(left[i], right[i]) != result[i]", ["GetIterResult"] = "Helpers.Add(left[i], right[i])"}), + ("SveVecBinOpTest.template", new Dictionary { ["TestName"] = "Sve_Add_sbyte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Add", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "(sbyte)TestLibrary.Generator.GetSByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateIterResult"] = "Helpers.Add(left[i], right[i]) != result[i]", ["GetIterResult"] = "Helpers.Add(left[i], right[i])"}), + ("SveVecBinOpTest.template", new Dictionary { ["TestName"] = "Sve_Add_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Add", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "(short)TestLibrary.Generator.GetInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["ValidateIterResult"] = "Helpers.Add(left[i], right[i]) != result[i]", ["GetIterResult"] = "Helpers.Add(left[i], right[i])"}), + ("SveVecBinOpTest.template", new Dictionary { ["TestName"] = "Sve_Add_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Add", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["ValidateIterResult"] = "Helpers.Add(left[i], right[i]) != result[i]", ["GetIterResult"] = "Helpers.Add(left[i], right[i])"}), + ("SveVecBinOpTest.template", new Dictionary { ["TestName"] = "Sve_Add_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Add", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["ValidateIterResult"] = "Helpers.Add(left[i], right[i]) != result[i]", ["GetIterResult"] = "Helpers.Add(left[i], right[i])"}), + ("SveVecBinOpTest.template", new Dictionary { ["TestName"] = "Sve_Add_byte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Add", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "(byte)TestLibrary.Generator.GetByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ValidateIterResult"] = "Helpers.Add(left[i], right[i]) != result[i]", ["GetIterResult"] = "Helpers.Add(left[i], right[i])"}), + ("SveVecBinOpTest.template", new Dictionary { ["TestName"] = "Sve_Add_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Add", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateIterResult"] = "Helpers.Add(left[i], right[i]) != result[i]", ["GetIterResult"] = "Helpers.Add(left[i], right[i])"}), + ("SveVecBinOpTest.template", new Dictionary { ["TestName"] = "Sve_Add_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Add", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateIterResult"] = "Helpers.Add(left[i], right[i]) != result[i]", ["GetIterResult"] = "Helpers.Add(left[i], right[i])"}), + ("SveVecBinOpTest.template", new Dictionary { ["TestName"] = "Sve_Add_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Add", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["ValidateIterResult"] = "Helpers.Add(left[i], right[i]) != result[i]", ["GetIterResult"] = "Helpers.Add(left[i], right[i])"}), + + ("SveConditionalSelect.template", new Dictionary { ["TestName"] = "Sve_ConditionalSelect_float", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ConditionalSelect", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp3"] = "TestLibrary.Generator.GetSingle()", ["ValidateIterResult"] = "(firstOp[i] != 0 ? (result[i] != secondOp[i]) : (result[i] != thirdOp[i]))",}), + ("SveConditionalSelect.template", new Dictionary { ["TestName"] = "Sve_ConditionalSelect_double", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ConditionalSelect", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp3"] = "TestLibrary.Generator.GetDouble()", ["ValidateIterResult"] = "(firstOp[i] != 0 ? (result[i] != secondOp[i]) : (result[i] != thirdOp[i]))",}), + ("SveConditionalSelect.template", new Dictionary { ["TestName"] = "Sve_ConditionalSelect_sbyte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ConditionalSelect", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["NextValueOp3"] = "TestLibrary.Generator.GetSByte()", ["ValidateIterResult"] = "(firstOp[i] != 0 ? (result[i] != secondOp[i]) : (result[i] != thirdOp[i]))",}), + ("SveConditionalSelect.template", new Dictionary { ["TestName"] = "Sve_ConditionalSelect_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ConditionalSelect", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int16", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt16()", ["ValidateIterResult"] = "(firstOp[i] != 0 ? (result[i] != secondOp[i]) : (result[i] != thirdOp[i]))",}), + ("SveConditionalSelect.template", new Dictionary { ["TestName"] = "Sve_ConditionalSelect_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ConditionalSelect", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()", ["ValidateIterResult"] = "(firstOp[i] != 0 ? (result[i] != secondOp[i]) : (result[i] != thirdOp[i]))",}), + ("SveConditionalSelect.template", new Dictionary { ["TestName"] = "Sve_ConditionalSelect_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ConditionalSelect", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()", ["ValidateIterResult"] = "(firstOp[i] != 0 ? (result[i] != secondOp[i]) : (result[i] != thirdOp[i]))",}), + ("SveConditionalSelect.template", new Dictionary { ["TestName"] = "Sve_ConditionalSelect_byte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ConditionalSelect", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Byte", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["NextValueOp3"] = "TestLibrary.Generator.GetByte()", ["ValidateIterResult"] = "(firstOp[i] != 0 ? (result[i] != secondOp[i]) : (result[i] != thirdOp[i]))",}), + ("SveConditionalSelect.template", new Dictionary { ["TestName"] = "Sve_ConditionalSelect_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ConditionalSelect", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt16", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt16()", ["ValidateIterResult"] = "(firstOp[i] != 0 ? (result[i] != secondOp[i]) : (result[i] != thirdOp[i]))",}), + ("SveConditionalSelect.template", new Dictionary { ["TestName"] = "Sve_ConditionalSelect_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ConditionalSelect", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()", ["ValidateIterResult"] = "(firstOp[i] != 0 ? (result[i] != secondOp[i]) : (result[i] != thirdOp[i]))",}), + ("SveConditionalSelect.template", new Dictionary { ["TestName"] = "Sve_ConditionalSelect_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ConditionalSelect", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()", ["ValidateIterResult"] = "(firstOp[i] != 0 ? (result[i] != secondOp[i]) : (result[i] != thirdOp[i]))",}), + + ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_float", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_double", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_sbyte", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_short", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_int", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_long", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_byte", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_ushort", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_uint", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_ulong", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), }; diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveConditionalSelect.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveConditionalSelect.template new file mode 100644 index 0000000000000..963fa1713eec3 --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveConditionalSelect.template @@ -0,0 +1,362 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +/****************************************************************************** + * This file is auto-generated from a template file by the GenerateTests.csx * + * script in tests\src\JIT\HardwareIntrinsics.Arm\Shared. In order to make * + * changes, please update the corresponding template and run according to the * + * directions listed in the file. * + ******************************************************************************/ + +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using System.Numerics; +using Xunit; + +namespace JIT.HardwareIntrinsics.Arm +{ + public static partial class Program + { + [Fact] + public static void {TestName}() + { + var test = new TernaryOpTest__{TestName}(); + + if (test.IsSupported) + { + // Validates basic functionality works, using Unsafe.Read + test.RunBasicScenario_UnsafeRead(); + + if ({LoadIsa}.IsSupported) + { + // Validates basic functionality works, using Load + test.RunBasicScenario_Load(); + } + + // Validates calling via reflection works, using Unsafe.Read + test.RunReflectionScenario_UnsafeRead(); + + // Validates passing a local works, using Unsafe.Read + test.RunLclVarScenario_UnsafeRead(); + + // Validates passing an instance member of a class works + test.RunClassFldScenario(); + + // Validates passing the field of a local struct works + test.RunStructLclFldScenario(); + + // Validates passing an instance member of a struct works + test.RunStructFldScenario(); + } + else + { + // Validates we throw on unsupported hardware + test.RunUnsupportedScenario(); + } + + if (!test.Succeeded) + { + throw new Exception("One or more scenarios did not complete as expected."); + } + } + } + + public sealed unsafe class TernaryOpTest__{TestName} + { + private struct DataTable + { + private byte[] inArray1; + private byte[] inArray2; + private byte[] inArray3; + private byte[] outArray; + + private GCHandle inHandle1; + private GCHandle inHandle2; + private GCHandle inHandle3; + private GCHandle outHandle; + + private ulong alignment; + + public DataTable({Op1BaseType}[] inArray1, {Op2BaseType}[] inArray2, {Op3BaseType}[] inArray3, {RetBaseType}[] outArray, int alignment) + { + int sizeOfinArray1 = inArray1.Length * Unsafe.SizeOf<{Op1BaseType}>(); + int sizeOfinArray2 = inArray2.Length * Unsafe.SizeOf<{Op2BaseType}>(); + int sizeOfinArray3 = inArray3.Length * Unsafe.SizeOf<{Op3BaseType}>(); + int sizeOfoutArray = outArray.Length * Unsafe.SizeOf<{RetBaseType}>(); + if ((alignment != 64 && alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfinArray1 || (alignment * 2) < sizeOfinArray2 || (alignment * 2) < sizeOfinArray3 || (alignment * 2) < sizeOfoutArray) + { + throw new ArgumentException($"Invalid value of alignment: {alignment}, sizeOfinArray1: {sizeOfinArray1}, sizeOfinArray2: {sizeOfinArray2}, sizeOfinArray3: {sizeOfinArray3}, sizeOfoutArray: {sizeOfoutArray}"); + } + + this.inArray1 = new byte[alignment * 2]; + this.inArray2 = new byte[alignment * 2]; + this.inArray3 = new byte[alignment * 2]; + this.outArray = new byte[alignment * 2]; + + this.inHandle1 = GCHandle.Alloc(this.inArray1, GCHandleType.Pinned); + this.inHandle2 = GCHandle.Alloc(this.inArray2, GCHandleType.Pinned); + this.inHandle3 = GCHandle.Alloc(this.inArray3, GCHandleType.Pinned); + this.outHandle = GCHandle.Alloc(this.outArray, GCHandleType.Pinned); + + this.alignment = (ulong)alignment; + + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray1Ptr), ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), (uint)sizeOfinArray1); + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray2Ptr), ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), (uint)sizeOfinArray2); + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray3Ptr), ref Unsafe.As<{Op3BaseType}, byte>(ref inArray3[0]), (uint)sizeOfinArray3); + } + + public void* inArray1Ptr => Align((byte*)(inHandle1.AddrOfPinnedObject().ToPointer()), alignment); + public void* inArray2Ptr => Align((byte*)(inHandle2.AddrOfPinnedObject().ToPointer()), alignment); + public void* inArray3Ptr => Align((byte*)(inHandle3.AddrOfPinnedObject().ToPointer()), alignment); + public void* outArrayPtr => Align((byte*)(outHandle.AddrOfPinnedObject().ToPointer()), alignment); + + public void Dispose() + { + inHandle1.Free(); + inHandle2.Free(); + inHandle3.Free(); + outHandle.Free(); + } + + private static unsafe void* Align(byte* buffer, ulong expectedAlignment) + { + return (void*)(((ulong)buffer + expectedAlignment - 1) & ~(expectedAlignment - 1)); + } + } + + private struct TestStruct + { + public {Op1VectorType}<{Op1BaseType}> _fld1; + public {Op2VectorType}<{Op2BaseType}> _fld2; + public {Op3VectorType}<{Op3BaseType}> _fld3; + + public static TestStruct Create() + { + var testStruct = new TestStruct(); + + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref testStruct._fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + for (var i = 0; i < Op2ElementCount; i++) { _data2[i] = {NextValueOp2}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2VectorType}<{Op2BaseType}>, byte>(ref testStruct._fld2), ref Unsafe.As<{Op2BaseType}, byte>(ref _data2[0]), (uint)Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>()); + for (var i = 0; i < Op3ElementCount; i++) { _data3[i] = {NextValueOp3}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op3VectorType}<{Op3BaseType}>, byte>(ref testStruct._fld3), ref Unsafe.As<{Op3BaseType}, byte>(ref _data3[0]), (uint)Unsafe.SizeOf<{Op3VectorType}<{Op3BaseType}>>()); + + return testStruct; + } + + public void RunStructFldScenario(TernaryOpTest__{TestName} testClass) + { + var result = {Isa}.{Method}(_fld1, _fld2, _fld3); + + Unsafe.Write(testClass._dataTable.outArrayPtr, result); + testClass.ValidateResult(_fld1, _fld2, _fld3, testClass._dataTable.outArrayPtr); + } + } + + private static readonly int LargestVectorSize = {LargestVectorSize}; + + private static readonly int Op1ElementCount = Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>() / sizeof({Op1BaseType}); + private static readonly int Op2ElementCount = Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>() / sizeof({Op2BaseType}); + private static readonly int Op3ElementCount = Unsafe.SizeOf<{Op3VectorType}<{Op3BaseType}>>() / sizeof({Op3BaseType}); + private static readonly int RetElementCount = Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>() / sizeof({RetBaseType}); + + private static {Op1BaseType}[] _data1 = new {Op1BaseType}[Op1ElementCount]; + private static {Op2BaseType}[] _data2 = new {Op2BaseType}[Op2ElementCount]; + private static {Op3BaseType}[] _data3 = new {Op3BaseType}[Op3ElementCount]; + + private {Op1VectorType}<{Op1BaseType}> _fld1; + private {Op2VectorType}<{Op2BaseType}> _fld2; + private {Op3VectorType}<{Op3BaseType}> _fld3; + + private DataTable _dataTable; + + public TernaryOpTest__{TestName}() + { + Succeeded = true; + + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + for (var i = 0; i < Op2ElementCount; i++) { _data2[i] = {NextValueOp2}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2VectorType}<{Op2BaseType}>, byte>(ref _fld2), ref Unsafe.As<{Op2BaseType}, byte>(ref _data2[0]), (uint)Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>()); + for (var i = 0; i < Op3ElementCount; i++) { _data3[i] = {NextValueOp3}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op3VectorType}<{Op3BaseType}>, byte>(ref _fld3), ref Unsafe.As<{Op3BaseType}, byte>(ref _data3[0]), (uint)Unsafe.SizeOf<{Op3VectorType}<{Op3BaseType}>>()); + + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + for (var i = 0; i < Op2ElementCount; i++) { _data2[i] = {NextValueOp2}; } + for (var i = 0; i < Op3ElementCount; i++) { _data3[i] = {NextValueOp3}; } + _dataTable = new DataTable(_data1, _data2, _data3, new {RetBaseType}[RetElementCount], LargestVectorSize); + } + + public bool IsSupported => {Isa}.IsSupported; + + public bool Succeeded { get; set; } + + public void RunBasicScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_UnsafeRead)); + + var result = {Isa}.{Method}( + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr), + Unsafe.Read<{Op2VectorType}<{Op2BaseType}>>(_dataTable.inArray2Ptr), + Unsafe.Read<{Op3VectorType}<{Op3BaseType}>>(_dataTable.inArray3Ptr) + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.inArray3Ptr, _dataTable.outArrayPtr); + } + + public void RunBasicScenario_Load() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_Load)); + + //TODO-SVE: Once register allocation exists for predicates, move loadMask into DataTable + {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateTrueMask{RetBaseType}(SveMaskPattern.All); + + var result = {Isa}.{Method}( + {LoadIsa}.Load{Op1VectorType}(loadMask, ({Op1BaseType}*)(_dataTable.inArray1Ptr)), + {LoadIsa}.Load{Op2VectorType}(loadMask, ({Op2BaseType}*)(_dataTable.inArray2Ptr)), + {LoadIsa}.Load{Op3VectorType}(loadMask, ({Op3BaseType}*)(_dataTable.inArray3Ptr)) + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.inArray3Ptr, _dataTable.outArrayPtr); + } + + public void RunReflectionScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunReflectionScenario_UnsafeRead)); + + var result = typeof({Isa}).GetMethod(nameof({Isa}.{Method}), new Type[] { typeof({Op1VectorType}<{Op1BaseType}>), typeof({Op2VectorType}<{Op2BaseType}>), typeof({Op3VectorType}<{Op3BaseType}>) }) + .Invoke(null, new object[] { + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr), + Unsafe.Read<{Op2VectorType}<{Op2BaseType}>>(_dataTable.inArray2Ptr), + Unsafe.Read<{Op3VectorType}<{Op3BaseType}>>(_dataTable.inArray3Ptr) + }); + + Unsafe.Write(_dataTable.outArrayPtr, ({RetVectorType}<{RetBaseType}>)(result)); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.inArray3Ptr, _dataTable.outArrayPtr); + } + + public void RunLclVarScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunLclVarScenario_UnsafeRead)); + + var op1 = Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr); + var op2 = Unsafe.Read<{Op2VectorType}<{Op2BaseType}>>(_dataTable.inArray2Ptr); + var op3 = Unsafe.Read<{Op3VectorType}<{Op3BaseType}>>(_dataTable.inArray3Ptr); + var result = {Isa}.{Method}(op1, op2, op3); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(op1, op2, op3, _dataTable.outArrayPtr); + } + + public void RunClassFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunClassFldScenario)); + + var result = {Isa}.{Method}(_fld1, _fld2, _fld3); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_fld1, _fld2, _fld3, _dataTable.outArrayPtr); + } + + public void RunStructLclFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructLclFldScenario)); + + var test = TestStruct.Create(); + var result = {Isa}.{Method}(test._fld1, test._fld2, test._fld3); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(test._fld1, test._fld2, test._fld3, _dataTable.outArrayPtr); + } + + public void RunStructFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructFldScenario)); + + var test = TestStruct.Create(); + test.RunStructFldScenario(this); + } + + public void RunUnsupportedScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunUnsupportedScenario)); + + bool succeeded = false; + + try + { + RunBasicScenario_UnsafeRead(); + } + catch (PlatformNotSupportedException) + { + succeeded = true; + } + + if (!succeeded) + { + Succeeded = false; + } + } + + private void ValidateResult({Op1VectorType}<{Op1BaseType}> op1, {Op2VectorType}<{Op2BaseType}> op2, {Op3VectorType}<{Op3BaseType}> op3, void* result, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] inArray2 = new {Op2BaseType}[Op2ElementCount]; + {Op3BaseType}[] inArray3 = new {Op3BaseType}[Op3ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), op1); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), op2); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op3BaseType}, byte>(ref inArray3[0]), op3); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateResult(inArray1, inArray2, inArray3, outArray, method); + } + + private void ValidateResult(void* op1, void* op2, void* op3, void* result, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] inArray2 = new {Op2BaseType}[Op2ElementCount]; + {Op3BaseType}[] inArray3 = new {Op3BaseType}[Op3ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), ref Unsafe.AsRef(op1), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), ref Unsafe.AsRef(op2), (uint)Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op3BaseType}, byte>(ref inArray3[0]), ref Unsafe.AsRef(op3), (uint)Unsafe.SizeOf<{Op3VectorType}<{Op3BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateResult(inArray1, inArray2, inArray3, outArray, method); + } + + private void ValidateResult({Op1BaseType}[] firstOp, {Op2BaseType}[] secondOp, {Op3BaseType}[] thirdOp, {RetBaseType}[] result, [CallerMemberName] string method = "") + { + bool succeeded = true; + + for (var i = 0; i < RetElementCount; i++) + { + if ({ValidateIterResult}) + { + succeeded = false; + break; + } + } + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1VectorType}<{Op1BaseType}>, {Op2VectorType}<{Op2BaseType}>, {Op3VectorType}<{Op3BaseType}>): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})"); + TestLibrary.TestFramework.LogInformation($"secondOp: ({string.Join(", ", secondOp)})"); + TestLibrary.TestFramework.LogInformation($" thirdOp: ({string.Join(", ", thirdOp)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + } +} diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadMaskedUnOpTest.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadMaskedUnOpTest.template index 09aaf2f442e13..d5a62e5cda1cf 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadMaskedUnOpTest.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadMaskedUnOpTest.template @@ -64,9 +64,9 @@ namespace JIT.HardwareIntrinsics.Arm { int sizeOfinArray1 = inArray1.Length * Unsafe.SizeOf<{Op2BaseType}>(); int sizeOfoutArray = outArray.Length * Unsafe.SizeOf<{RetBaseType}>(); - if ((alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfinArray1 || (alignment * 2) < sizeOfoutArray) + if ((alignment != 64 && alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfinArray1 || (alignment * 2) < sizeOfoutArray) { - throw new ArgumentException("Invalid value of alignment"); + throw new ArgumentException($"Invalid value of alignment: {alignment}, sizeOfinArray1: {sizeOfinArray1}, sizeOfoutArray: {sizeOfoutArray}"); } this.inArray1 = new byte[alignment * 2]; diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveBinaryOpTestTemplate.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveBinaryOpTestTemplate.template new file mode 100644 index 0000000000000..bd15acab428a0 --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveBinaryOpTestTemplate.template @@ -0,0 +1,435 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +/****************************************************************************** + * This file is auto-generated from a template file by the GenerateTests.csx * + * script in tests\src\JIT\HardwareIntrinsics.Arm\Shared. In order to make * + * changes, please update the corresponding template and run according to the * + * directions listed in the file. * + ******************************************************************************/ + +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using Xunit; + +namespace JIT.HardwareIntrinsics.Arm +{ + public static partial class Program + { + [Fact] + public static void {TestName}() + { + var test = new {TemplateName}BinaryOpTest__{TestName}(); + + if (test.IsSupported) + { + // Validates basic functionality works, using Unsafe.Read + test.RunBasicScenario_UnsafeRead(); + + if ({LoadIsa}.IsSupported) + { + // Validates basic functionality works, using Load + test.RunBasicScenario_Load(); + } + + // Validates calling via reflection works, using Unsafe.Read + test.RunReflectionScenario_UnsafeRead(); + + // Validates passing a local works, using Unsafe.Read + test.RunLclVarScenario_UnsafeRead(); + + // Validates passing an instance member of a class works + test.RunClassFldScenario(); + + // Validates passing the field of a local struct works + test.RunStructLclFldScenario(); + + // Validates passing an instance member of a struct works + test.RunStructFldScenario(); + + // Validates executing the test inside conditional, with op1 as falseValue + test.ConditionalSelect_Op1(); + + // Validates executing the test inside conditional, with op2 as falseValue + test.ConditionalSelect_Op2(); + + // Validates executing the test inside conditional, with op3 as falseValue + test.ConditionalSelect_FalseOp(); + + // Validates executing the test inside conditional, with op3 as zero + test.ConditionalSelect_ZeroOp(); + } + else + { + // Validates we throw on unsupported hardware + test.RunUnsupportedScenario(); + } + + if (!test.Succeeded) + { + throw new Exception("One or more scenarios did not complete as expected."); + } + } + } + + public sealed unsafe class {TemplateName}BinaryOpTest__{TestName} + { + private struct DataTable + { + private byte[] inArray1; + private byte[] inArray2; + private byte[] outArray; + + private GCHandle inHandle1; + private GCHandle inHandle2; + private GCHandle outHandle; + + private ulong alignment; + + public DataTable({Op1BaseType}[] inArray1, {Op2BaseType}[] inArray2, {RetBaseType}[] outArray, int alignment) + { + int sizeOfinArray1 = inArray1.Length * Unsafe.SizeOf<{Op1BaseType}>(); + int sizeOfinArray2 = inArray2.Length * Unsafe.SizeOf<{Op2BaseType}>(); + int sizeOfoutArray = outArray.Length * Unsafe.SizeOf<{RetBaseType}>(); + if ((alignment != 64 && alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfinArray1 || (alignment * 2) < sizeOfinArray2 || (alignment * 2) < sizeOfoutArray) + { + throw new ArgumentException($"Invalid value of alignment: {alignment}, sizeOfinArray1: {sizeOfinArray1}, sizeOfinArray2: {sizeOfinArray2}, sizeOfoutArray: {sizeOfoutArray}"); + } + + this.inArray1 = new byte[alignment * 2]; + this.inArray2 = new byte[alignment * 2]; + this.outArray = new byte[alignment * 2]; + + this.inHandle1 = GCHandle.Alloc(this.inArray1, GCHandleType.Pinned); + this.inHandle2 = GCHandle.Alloc(this.inArray2, GCHandleType.Pinned); + this.outHandle = GCHandle.Alloc(this.outArray, GCHandleType.Pinned); + + this.alignment = (ulong)alignment; + + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray1Ptr), ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), (uint)sizeOfinArray1); + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray2Ptr), ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), (uint)sizeOfinArray2); + } + + public void* inArray1Ptr => Align((byte*)(inHandle1.AddrOfPinnedObject().ToPointer()), alignment); + public void* inArray2Ptr => Align((byte*)(inHandle2.AddrOfPinnedObject().ToPointer()), alignment); + public void* outArrayPtr => Align((byte*)(outHandle.AddrOfPinnedObject().ToPointer()), alignment); + + public void Dispose() + { + inHandle1.Free(); + inHandle2.Free(); + outHandle.Free(); + } + + private static unsafe void* Align(byte* buffer, ulong expectedAlignment) + { + return (void*)(((ulong)buffer + expectedAlignment - 1) & ~(expectedAlignment - 1)); + } + } + + private struct TestStruct + { + public {Op1VectorType}<{Op1BaseType}> _fld1; + public {Op2VectorType}<{Op2BaseType}> _fld2; + + public static TestStruct Create() + { + var testStruct = new TestStruct(); + + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref testStruct._fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + for (var i = 0; i < Op2ElementCount; i++) { _data2[i] = {NextValueOp2}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2VectorType}<{Op2BaseType}>, byte>(ref testStruct._fld2), ref Unsafe.As<{Op2BaseType}, byte>(ref _data2[0]), (uint)Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>()); + + return testStruct; + } + + public void RunStructFldScenario({TemplateName}BinaryOpTest__{TestName} testClass) + { + var result = {Isa}.{Method}(_fld1, _fld2); + + Unsafe.Write(testClass._dataTable.outArrayPtr, result); + testClass.ValidateResult(_fld1, _fld2, testClass._dataTable.outArrayPtr); + } + } + + private static readonly int LargestVectorSize = {LargestVectorSize}; + + private static readonly int Op1ElementCount = Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>() / sizeof({Op1BaseType}); + private static readonly int Op2ElementCount = Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>() / sizeof({Op2BaseType}); + private static readonly int RetElementCount = Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>() / sizeof({RetBaseType}); + + private static {Op1BaseType}[] _maskData = new {Op1BaseType}[Op1ElementCount]; + private static {Op1BaseType}[] _data1 = new {Op1BaseType}[Op1ElementCount]; + private static {Op2BaseType}[] _data2 = new {Op2BaseType}[Op2ElementCount]; + + private {Op1VectorType}<{Op1BaseType}> _mask; + private {Op1VectorType}<{Op1BaseType}> _fld1; + private {Op2VectorType}<{Op2BaseType}> _fld2; + private {Op2VectorType}<{Op2BaseType}> _falseFld; + + private DataTable _dataTable; + + public {TemplateName}BinaryOpTest__{TestName}() + { + Succeeded = true; + + for (var i = 0; i < Op1ElementCount; i++) { _maskData[i] = ({Op1BaseType})({NextValueOp1} % 2); } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _mask), ref Unsafe.As<{Op1BaseType}, byte>(ref _maskData[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + for (var i = 0; i < Op2ElementCount; i++) { _data2[i] = {NextValueOp2}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2VectorType}<{Op2BaseType}>, byte>(ref _fld2), ref Unsafe.As<{Op2BaseType}, byte>(ref _data2[0]), (uint)Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2VectorType}<{Op2BaseType}>, byte>(ref _falseFld), ref Unsafe.As<{Op2BaseType}, byte>(ref _data2[0]), (uint)Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>()); + + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + for (var i = 0; i < Op2ElementCount; i++) { _data2[i] = {NextValueOp2}; } + _dataTable = new DataTable(_data1, _data2, new {RetBaseType}[RetElementCount], LargestVectorSize); + } + + public bool IsSupported => {Isa}.IsSupported; + + public bool Succeeded { get; set; } + + public void RunBasicScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_UnsafeRead)); + + var result = {Isa}.{Method}( + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr), + Unsafe.Read<{Op2VectorType}<{Op2BaseType}>>(_dataTable.inArray2Ptr) + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.outArrayPtr); + } + + public void RunBasicScenario_Load() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_Load)); + + {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateTrueMask{RetBaseType}(SveMaskPattern.All); + + var result = {Isa}.{Method}( + {LoadIsa}.Load{Op1VectorType}(loadMask, ({Op1BaseType}*)(_dataTable.inArray1Ptr)), + {LoadIsa}.Load{Op2VectorType}(loadMask, ({Op2BaseType}*)(_dataTable.inArray2Ptr)) + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.outArrayPtr); + } + + public void RunReflectionScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunReflectionScenario_UnsafeRead)); + + var result = typeof({Isa}).GetMethod(nameof({Isa}.{Method}), new Type[] { typeof({Op1VectorType}<{Op1BaseType}>), typeof({Op2VectorType}<{Op2BaseType}>) }) + .Invoke(null, new object[] { + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr), + Unsafe.Read<{Op2VectorType}<{Op2BaseType}>>(_dataTable.inArray2Ptr) + }); + + Unsafe.Write(_dataTable.outArrayPtr, ({RetVectorType}<{RetBaseType}>)(result)); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.outArrayPtr); + } + + public void RunLclVarScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunLclVarScenario_UnsafeRead)); + + var op1 = Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr); + var op2 = Unsafe.Read<{Op2VectorType}<{Op2BaseType}>>(_dataTable.inArray2Ptr); + var result = {Isa}.{Method}(op1, op2); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(op1, op2, _dataTable.outArrayPtr); + } + + public void RunClassFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunClassFldScenario)); + + var result = {Isa}.{Method}(_fld1, _fld2); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_fld1, _fld2, _dataTable.outArrayPtr); + } + + public void RunStructLclFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructLclFldScenario)); + + var test = TestStruct.Create(); + var result = {Isa}.{Method}(test._fld1, test._fld2); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(test._fld1, test._fld2, _dataTable.outArrayPtr); + } + + public void RunStructFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructFldScenario)); + + var test = TestStruct.Create(); + test.RunStructFldScenario(this); + } + + public void ConditionalSelect_Op1() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_mask"); + ConditionalSelectScenario(_mask, _fld1, _fld2, _fld1); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_zero"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld1); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_all"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld1); + } + + public void ConditionalSelect_Op2() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op2_mask"); + ConditionalSelectScenario(_mask, _fld1, _fld2, _fld2); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op2_zero"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld2); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op2_all"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld2); + } + + public void ConditionalSelect_FalseOp() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_mask"); + ConditionalSelectScenario(_mask, _fld1, _fld2, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_zero"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_all"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _falseFld); + } + + public void ConditionalSelect_ZeroOp() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_mask"); + ConditionalSelectScenario(_mask, _fld1, _fld2, {Op1VectorType}<{Op1BaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, {Op1VectorType}<{Op1BaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, {Op1VectorType}<{Op1BaseType}>.Zero); + } + + [method: MethodImpl(MethodImplOptions.AggressiveInlining)] + private void ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op1VectorType}<{Op1BaseType}> op2, {Op1VectorType}<{Op1BaseType}> falseOp) + { + var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2), falseOp); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateConditionalSelectResult(mask, op1, op2, falseOp, _dataTable.outArrayPtr); + } + + public void RunUnsupportedScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunUnsupportedScenario)); + + bool succeeded = false; + + try + { + RunBasicScenario_UnsafeRead(); + } + catch (PlatformNotSupportedException) + { + succeeded = true; + } + + if (!succeeded) + { + Succeeded = false; + } + } + + private void ValidateConditionalSelectResult({Op1VectorType}<{Op1BaseType}> maskOp, {Op1VectorType}<{Op1BaseType}> leftOp, {Op1VectorType}<{Op1BaseType}> rightOp, {Op1VectorType}<{Op1BaseType}> falseOp, void* output, [CallerMemberName] string method = "") + { + {Op1BaseType}[] mask = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] left = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] right = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] falseVal = new {Op1BaseType}[Op1ElementCount]; + {RetBaseType}[] result = new {RetBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref mask[0]), maskOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref left[0]), leftOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref right[0]), rightOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref falseVal[0]), falseOp); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref result[0]), ref Unsafe.AsRef(output), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + bool succeeded = true; + + {TemplateValidationLogicForCndSel} + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1VectorType}<{Op1BaseType}>, {Op2VectorType}<{Op2BaseType}>): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" mask: ({string.Join(", ", mask)})"); + TestLibrary.TestFramework.LogInformation($" left: ({string.Join(", ", left)})"); + TestLibrary.TestFramework.LogInformation($" right: ({string.Join(", ", right)})"); + TestLibrary.TestFramework.LogInformation($" falseOp: ({string.Join(", ", falseVal)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + + private void ValidateResult({Op1VectorType}<{Op1BaseType}> op1, {Op2VectorType}<{Op2BaseType}> op2, void* result, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] inArray2 = new {Op2BaseType}[Op2ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), op1); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), op2); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateResult(inArray1, inArray2, outArray, method); + } + + private void ValidateResult(void* op1, void* op2, void* result, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] inArray2 = new {Op2BaseType}[Op2ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), ref Unsafe.AsRef(op1), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), ref Unsafe.AsRef(op2), (uint)Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateResult(inArray1, inArray2, outArray, method); + } + + private void ValidateResult({Op1BaseType}[] left, {Op2BaseType}[] right, {RetBaseType}[] result, [CallerMemberName] string method = "") + { + bool succeeded = true; + + {TemplateValidationLogic} + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1VectorType}<{Op1BaseType}>, {Op2VectorType}<{Op2BaseType}>): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" left: ({string.Join(", ", left)})"); + TestLibrary.TestFramework.LogInformation($" right: ({string.Join(", ", right)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + } +} diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveUnaryOpTestTemplate.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveUnaryOpTestTemplate.template new file mode 100644 index 0000000000000..557442de4cf0b --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveUnaryOpTestTemplate.template @@ -0,0 +1,391 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +/****************************************************************************** + * This file is auto-generated from a template file by the GenerateTests.csx * + * script in tests\src\JIT\HardwareIntrinsics.Arm\Shared. In order to make * + * changes, please update the corresponding template and run according to the * + * directions listed in the file. * + ******************************************************************************/ + +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using Xunit; + +namespace JIT.HardwareIntrinsics.Arm +{ + public static partial class Program + { + [Fact] + public static void {TestName}() + { + var test = new {TemplateName}UnaryOpTest__{TestName}(); + + if (test.IsSupported) + { + // Validates basic functionality works, using Unsafe.Read + test.RunBasicScenario_UnsafeRead(); + + if ({LoadIsa}.IsSupported) + { + // Validates basic functionality works, using Load + test.RunBasicScenario_Load(); + } + + // Validates calling via reflection works, using Unsafe.Read + test.RunReflectionScenario_UnsafeRead(); + + // Validates passing a local works, using Unsafe.Read + test.RunLclVarScenario_UnsafeRead(); + + // Validates passing an instance member of a class works + test.RunClassFldScenario(); + + // Validates passing the field of a local struct works + test.RunStructLclFldScenario(); + + // Validates passing an instance member of a struct works + test.RunStructFldScenario(); + + // Validates executing the test inside conditional, with op1 as falseValue + test.ConditionalSelect_Op1(); + + // Validates executing the test inside conditional, with op3 as falseValue + test.ConditionalSelect_FalseOp(); + + // Validates executing the test inside conditional, with op3 as zero + test.ConditionalSelect_ZeroOp(); + } + else + { + // Validates we throw on unsupported hardware + test.RunUnsupportedScenario(); + } + + if (!test.Succeeded) + { + throw new Exception("One or more scenarios did not complete as expected."); + } + } + } + + public sealed unsafe class {TemplateName}UnaryOpTest__{TestName} + { + private struct DataTable + { + private byte[] inArray1; + private byte[] outArray; + + private GCHandle inHandle1; + private GCHandle outHandle; + + private ulong alignment; + + public DataTable({Op1BaseType}[] inArray1, {RetBaseType}[] outArray, int alignment) + { + int sizeOfinArray1 = inArray1.Length * Unsafe.SizeOf<{Op1BaseType}>(); + int sizeOfoutArray = outArray.Length * Unsafe.SizeOf<{RetBaseType}>(); + if ((alignment != 64 && alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfinArray1 || (alignment * 2) < sizeOfoutArray) + { + throw new ArgumentException($"Invalid value of alignment: {alignment}, sizeOfinArray1: {sizeOfinArray1}, sizeOfoutArray: {sizeOfoutArray}"); + } + + this.inArray1 = new byte[alignment * 2]; + this.outArray = new byte[alignment * 2]; + + this.inHandle1 = GCHandle.Alloc(this.inArray1, GCHandleType.Pinned); + this.outHandle = GCHandle.Alloc(this.outArray, GCHandleType.Pinned); + + this.alignment = (ulong)alignment; + + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray1Ptr), ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), (uint)sizeOfinArray1); + } + + public void* inArray1Ptr => Align((byte*)(inHandle1.AddrOfPinnedObject().ToPointer()), alignment); + public void* outArrayPtr => Align((byte*)(outHandle.AddrOfPinnedObject().ToPointer()), alignment); + + public void Dispose() + { + inHandle1.Free(); + outHandle.Free(); + } + + private static unsafe void* Align(byte* buffer, ulong expectedAlignment) + { + return (void*)(((ulong)buffer + expectedAlignment - 1) & ~(expectedAlignment - 1)); + } + } + + private struct TestStruct + { + public {Op1VectorType}<{Op1BaseType}> _fld1; + + public static TestStruct Create() + { + var testStruct = new TestStruct(); + + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref testStruct._fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + + return testStruct; + } + + public void RunStructFldScenario({TemplateName}UnaryOpTest__{TestName} testClass) + { + var result = {Isa}.{Method}(_fld1); + + Unsafe.Write(testClass._dataTable.outArrayPtr, result); + testClass.ValidateResult(_fld1, testClass._dataTable.outArrayPtr); + } + } + + private static readonly int LargestVectorSize = {LargestVectorSize}; + + private static readonly int Op1ElementCount = Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>() / sizeof({Op1BaseType}); + private static readonly int RetElementCount = Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>() / sizeof({RetBaseType}); + + private static {Op1BaseType}[] _maskData = new {Op1BaseType}[Op1ElementCount]; + private static {Op1BaseType}[] _data1 = new {Op1BaseType}[Op1ElementCount]; + + private {Op1VectorType}<{Op1BaseType}> _mask; + private {Op1VectorType}<{Op1BaseType}> _fld1; + private {Op1VectorType}<{Op1BaseType}> _falseFld; + + private DataTable _dataTable; + + public {TemplateName}UnaryOpTest__{TestName}() + { + Succeeded = true; + + for (var i = 0; i < Op1ElementCount; i++) { _maskData[i] = ({Op1BaseType})({NextValueOp1} % 2); } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _mask), ref Unsafe.As<{Op1BaseType}, byte>(ref _maskData[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _falseFld), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + _dataTable = new DataTable(_data1, new {RetBaseType}[RetElementCount], LargestVectorSize); + } + + public bool IsSupported => {Isa}.IsSupported; + + public bool Succeeded { get; set; } + + public void RunBasicScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_UnsafeRead)); + + var result = {Isa}.{Method}( + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr) + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.outArrayPtr); + } + + public void RunBasicScenario_Load() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_Load)); + + {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateTrueMask{RetBaseType}(SveMaskPattern.All); + + var result = {Isa}.{Method}( + {LoadIsa}.Load{Op1VectorType}(loadMask, ({Op1BaseType}*)(_dataTable.inArray1Ptr)) + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.outArrayPtr); + } + + public void RunReflectionScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunReflectionScenario_UnsafeRead)); + + var result = typeof({Isa}).GetMethod(nameof({Isa}.{Method}), new Type[] { typeof({Op1VectorType}<{Op1BaseType}>) }) + .Invoke(null, new object[] { + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr) + }); + + Unsafe.Write(_dataTable.outArrayPtr, ({RetVectorType}<{RetBaseType}>)(result)); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.outArrayPtr); + } + + public void RunLclVarScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunLclVarScenario_UnsafeRead)); + + var op1 = Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr); + var result = {Isa}.{Method}(op1); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(op1, _dataTable.outArrayPtr); + } + + public void RunClassFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunClassFldScenario)); + + var result = {Isa}.{Method}(_fld1); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_fld1, _dataTable.outArrayPtr); + } + + public void RunStructLclFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructLclFldScenario)); + + var test = TestStruct.Create(); + var result = {Isa}.{Method}(test._fld1); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(test._fld1, _dataTable.outArrayPtr); + } + + public void RunStructFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructFldScenario)); + + var test = TestStruct.Create(); + test.RunStructFldScenario(this); + } + + public void ConditionalSelect_Op1() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_mask"); + ConditionalSelectScenario(_mask, _fld1, _fld1); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_zero"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld1); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_all"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld1); + } + + public void ConditionalSelect_FalseOp() + { + TestLibrary.TestFramework.BeginScenario(nameof(ConditionalSelect_FalseOp)); + ConditionalSelectScenario(_mask, _fld1, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_zero"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_all"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _falseFld); + } + + public void ConditionalSelect_ZeroOp() + { + TestLibrary.TestFramework.BeginScenario(nameof(ConditionalSelect_ZeroOp)); + ConditionalSelectScenario(_mask, _fld1, {Op1VectorType}<{Op1BaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, {Op1VectorType}<{Op1BaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, {Op1VectorType}<{Op1BaseType}>.Zero); + } + + [method: MethodImpl(MethodImplOptions.AggressiveInlining)] + private void ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op1VectorType}<{Op1BaseType}> falseOp) + { + var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1), falseOp); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateConditionalSelectResult(mask, op1, falseOp, _dataTable.outArrayPtr); + } + + public void RunUnsupportedScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunUnsupportedScenario)); + + bool succeeded = false; + + try + { + RunBasicScenario_UnsafeRead(); + } + catch (PlatformNotSupportedException) + { + succeeded = true; + } + + if (!succeeded) + { + Succeeded = false; + } + } + + private void ValidateConditionalSelectResult({Op1VectorType}<{Op1BaseType}> maskOp, {Op1VectorType}<{Op1BaseType}> leftOp, {Op1VectorType}<{Op1BaseType}> falseOp, void* output, [CallerMemberName] string method = "") + { + {Op1BaseType}[] mask = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] left = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] falseVal = new {Op1BaseType}[Op1ElementCount]; + {RetBaseType}[] result = new {RetBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref mask[0]), maskOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref left[0]), leftOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref falseVal[0]), falseOp); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref result[0]), ref Unsafe.AsRef(output), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + bool succeeded = true; + + {TemplateValidationLogicForCndSel} + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1VectorType}<{Op1BaseType}>, {Op1VectorType}<{Op1BaseType}>): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" mask: ({string.Join(", ", mask)})"); + TestLibrary.TestFramework.LogInformation($" left: ({string.Join(", ", left)})"); + TestLibrary.TestFramework.LogInformation($" falseOp: ({string.Join(", ", falseVal)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + + private void ValidateResult({Op1VectorType}<{Op1BaseType}> op1, void* result, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), op1); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateResult(inArray1, outArray, method); + } + + private void ValidateResult(void* op1, void* result, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), ref Unsafe.AsRef(op1), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateResult(inArray1, outArray, method); + } + + private void ValidateResult({Op1BaseType}[] firstOp, {RetBaseType}[] result, [CallerMemberName] string method = "") + { + bool succeeded = true; + + {TemplateValidationLogic} + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1VectorType}<{Op1BaseType}>): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + } +}