From 6169e41a6ba28843d18a778a4e5e0edebed54414 Mon Sep 17 00:00:00 2001 From: Aman Khalid Date: Mon, 12 Aug 2024 17:07:49 +0000 Subject: [PATCH] ARM64-SVE: Add `GatherVectorWithByteOffsetFirstFaulting` (#106199) Co-authored-by: Jakob Botsch Nielsen --- src/coreclr/jit/gentree.cpp | 1 + src/coreclr/jit/hwintrinsic.cpp | 1 + src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 8 +- src/coreclr/jit/hwintrinsiclistarm64sve.h | 1 + src/coreclr/jit/lowerarmarch.cpp | 50 +- src/coreclr/jit/lsraarm64.cpp | 1 + .../Arm/Sve.PlatformNotSupported.cs | 77 +- .../src/System/Runtime/Intrinsics/Arm/Sve.cs | 77 +- .../ref/System.Runtime.Intrinsics.cs | 12 + .../GenerateHWIntrinsicTests_Arm.cs | 15 +- .../HardwareIntrinsics/Arm/Shared/Helpers.cs | 88 ++ ...therVectorByteOffsetFirstFaulting.template | 764 ++++++++++++++++++ 12 files changed, 1048 insertions(+), 47 deletions(-) create mode 100644 src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorByteOffsetFirstFaulting.template diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index c5fbcc8c5680f..aaf67c59530b0 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -26690,6 +26690,7 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const case NI_Sve_GatherVectorUInt16ZeroExtend: case NI_Sve_GatherVectorUInt32WithByteOffsetsZeroExtend: case NI_Sve_GatherVectorUInt32ZeroExtend: + case NI_Sve_GatherVectorWithByteOffsetFirstFaulting: case NI_Sve_GatherVectorWithByteOffsets: case NI_Sve_LoadVector: case NI_Sve_LoadVectorNonTemporal: diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index dbadcb8fb2266..0d56db1980df2 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -2185,6 +2185,7 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, case NI_Sve_GatherVectorUInt16ZeroExtend: case NI_Sve_GatherVectorUInt32WithByteOffsetsZeroExtend: case NI_Sve_GatherVectorUInt32ZeroExtend: + case NI_Sve_GatherVectorWithByteOffsetFirstFaulting: case NI_Sve_GatherVectorWithByteOffsets: assert(varTypeIsSIMD(op3->TypeGet())); if (numArgs == 3) diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index b57cda42c2f9d..e97d0fe1869d6 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -2090,14 +2090,16 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) case NI_Sve_GatherVectorUInt16ZeroExtend: case NI_Sve_GatherVectorUInt32WithByteOffsetsZeroExtend: case NI_Sve_GatherVectorUInt32ZeroExtend: + case NI_Sve_GatherVectorWithByteOffsetFirstFaulting: { if (!varTypeIsSIMD(intrin.op2->gtType)) { // GatherVector...(Vector mask, T* address, Vector indices) - emitAttr baseSize = emitActualTypeSize(intrin.baseType); - bool isLoadingBytes = ((ins == INS_sve_ld1b) || (ins == INS_sve_ld1sb) || (ins == INS_sve_ldff1b) || - (ins == INS_sve_ldff1sb)); + emitAttr baseSize = emitActualTypeSize(intrin.baseType); + bool isLoadingBytes = + ((ins == INS_sve_ld1b) || (ins == INS_sve_ld1sb) || (ins == INS_sve_ldff1b) || + (ins == INS_sve_ldff1sb) || (intrin.id == NI_Sve_GatherVectorWithByteOffsetFirstFaulting)); insScalableOpts sopt = INS_SCALABLE_OPTS_NONE; if (baseSize == EA_4BYTE) diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index 41a82b2fdaefd..687f3378d1de8 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -121,6 +121,7 @@ HARDWARE_INTRINSIC(Sve, GatherVectorUInt16WithByteOffsetsZeroExtend, HARDWARE_INTRINSIC(Sve, GatherVectorUInt16ZeroExtend, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_sve_ld1h, INS_sve_ld1h, INS_sve_ld1h, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, GatherVectorUInt32WithByteOffsetsZeroExtend, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, GatherVectorUInt32ZeroExtend, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, GatherVectorWithByteOffsetFirstFaulting, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialSideEffect_Other) HARDWARE_INTRINSIC(Sve, GatherVectorWithByteOffsets, -1, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1d, INS_sve_ld1d, INS_sve_ld1w, INS_sve_ld1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, GetActiveElementCount, -1, 2, {INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation) HARDWARE_INTRINSIC(Sve, GetFfrByte, -1, 0, {INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index 98f933c4c773e..baa7257f4a2c3 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -1782,6 +1782,14 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) break; } case NI_Sve_GatherVectorFirstFaulting: + case NI_Sve_GatherVectorWithByteOffsetFirstFaulting: + case NI_Sve_LoadVectorByteZeroExtendFirstFaulting: + case NI_Sve_LoadVectorFirstFaulting: + case NI_Sve_LoadVectorInt16SignExtendFirstFaulting: + case NI_Sve_LoadVectorInt32SignExtendFirstFaulting: + case NI_Sve_LoadVectorSByteSignExtendFirstFaulting: + case NI_Sve_LoadVectorUInt16ZeroExtendFirstFaulting: + case NI_Sve_LoadVectorUInt32ZeroExtendFirstFaulting: { LIR::Use use; bool foundUse = BlockRange().TryGetUse(node, &use); @@ -1825,47 +1833,6 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) StoreFFRValue(node); break; } - case NI_Sve_LoadVectorByteZeroExtendFirstFaulting: - case NI_Sve_LoadVectorFirstFaulting: - case NI_Sve_LoadVectorInt16SignExtendFirstFaulting: - case NI_Sve_LoadVectorInt32SignExtendFirstFaulting: - case NI_Sve_LoadVectorSByteSignExtendFirstFaulting: - case NI_Sve_LoadVectorUInt16ZeroExtendFirstFaulting: - case NI_Sve_LoadVectorUInt32ZeroExtendFirstFaulting: - { - LIR::Use use; - bool foundUse = BlockRange().TryGetUse(node, &use); - - if (m_ffrTrashed) - { - // Consume the FFR register value from local variable to simulate "use" of FFR, - // only if it was trashed. If it was not trashed, we do not have to reload the - // contents of the FFR register. - - unsigned lclNum = comp->getFFRegisterVarNum(); - GenTree* lclVar = comp->gtNewLclvNode(lclNum, TYP_MASK); - BlockRange().InsertBefore(node, lclVar); - LowerNode(lclVar); - - node->ResetHWIntrinsicId(intrinsicId, comp, node->Op(1), node->Op(2), lclVar); - } - - if (foundUse) - { - unsigned tmpNum = comp->lvaGrabTemp(true DEBUGARG("Return value result/FFR")); - LclVarDsc* tmpVarDsc = comp->lvaGetDesc(tmpNum); - tmpVarDsc->lvType = node->TypeGet(); - GenTree* storeLclVar; - use.ReplaceWithLclVar(comp, tmpNum, &storeLclVar); - } - else - { - node->SetUnusedValue(); - } - - StoreFFRValue(node); - break; - } default: break; } @@ -4146,6 +4113,7 @@ void Lowering::StoreFFRValue(GenTreeHWIntrinsic* node) switch (node->GetHWIntrinsicId()) { case NI_Sve_GatherVectorFirstFaulting: + case NI_Sve_GatherVectorWithByteOffsetFirstFaulting: case NI_Sve_LoadVectorByteZeroExtendFirstFaulting: case NI_Sve_LoadVectorFirstFaulting: case NI_Sve_LoadVectorInt16SignExtendFirstFaulting: diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index f0bc32e1f3da2..5c4a1b2f8edff 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -2113,6 +2113,7 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou case NI_Sve_GatherVectorUInt16ZeroExtend: case NI_Sve_GatherVectorUInt32WithByteOffsetsZeroExtend: case NI_Sve_GatherVectorUInt32ZeroExtend: + case NI_Sve_GatherVectorWithByteOffsetFirstFaulting: assert(intrinsicTree->OperIsMemoryLoadOrStore()); FALLTHROUGH; diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs index bcc541474ee42..ca2f20bc2f21d 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs @@ -4290,7 +4290,82 @@ internal Arm64() { } public static unsafe Vector GatherVectorUInt32ZeroExtend(Vector mask, uint* address, Vector indices) { throw new PlatformNotSupportedException(); } - // Unextended load + /// Unextended load, first-faulting + + /// + /// svfloat64_t svldff1_gather_[s64]offset[_f64](svbool_t pg, const float64_t *base, svint64_t offsets) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, Zoffsets.D] + /// + public static unsafe Vector GatherVectorWithByteOffsetFirstFaulting(Vector mask, double* address, Vector offsets) { throw new PlatformNotSupportedException(); } + + /// + /// svfloat64_t svldff1_gather_[u64]offset[_f64](svbool_t pg, const float64_t *base, svuint64_t offsets) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, Zoffsets.D] + /// + public static unsafe Vector GatherVectorWithByteOffsetFirstFaulting(Vector mask, double* address, Vector offsets) { throw new PlatformNotSupportedException(); } + + /// + /// svint32_t svldff1_gather_[s32]offset[_s32](svbool_t pg, const int32_t *base, svint32_t offsets) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, Zoffsets.S, SXTW] + /// + public static unsafe Vector GatherVectorWithByteOffsetFirstFaulting(Vector mask, int* address, Vector offsets) { throw new PlatformNotSupportedException(); } + + /// + /// svint32_t svldff1_gather_[u32]offset[_s32](svbool_t pg, const int32_t *base, svuint32_t offsets) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, Zoffsets.S, UXTW] + /// + public static unsafe Vector GatherVectorWithByteOffsetFirstFaulting(Vector mask, int* address, Vector offsets) { throw new PlatformNotSupportedException(); } + + /// + /// svint64_t svldff1_gather_[s64]offset[_s64](svbool_t pg, const int64_t *base, svint64_t offsets) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, Zoffsets.D] + /// + public static unsafe Vector GatherVectorWithByteOffsetFirstFaulting(Vector mask, long* address, Vector offsets) { throw new PlatformNotSupportedException(); } + + /// + /// svint64_t svldff1_gather_[u64]offset[_s64](svbool_t pg, const int64_t *base, svuint64_t offsets) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, Zoffsets.D] + /// + public static unsafe Vector GatherVectorWithByteOffsetFirstFaulting(Vector mask, long* address, Vector offsets) { throw new PlatformNotSupportedException(); } + + /// + /// svfloat32_t svldff1_gather_[s32]offset[_f32](svbool_t pg, const float32_t *base, svint32_t offsets) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, Zoffsets.S, SXTW] + /// + public static unsafe Vector GatherVectorWithByteOffsetFirstFaulting(Vector mask, float* address, Vector offsets) { throw new PlatformNotSupportedException(); } + + /// + /// svfloat32_t svldff1_gather_[u32]offset[_f32](svbool_t pg, const float32_t *base, svuint32_t offsets) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, Zoffsets.S, UXTW] + /// + public static unsafe Vector GatherVectorWithByteOffsetFirstFaulting(Vector mask, float* address, Vector offsets) { throw new PlatformNotSupportedException(); } + + /// + /// svuint32_t svldff1_gather_[s32]offset[_u32](svbool_t pg, const uint32_t *base, svint32_t offsets) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, Zoffsets.S, SXTW] + /// + public static unsafe Vector GatherVectorWithByteOffsetFirstFaulting(Vector mask, uint* address, Vector offsets) { throw new PlatformNotSupportedException(); } + + /// + /// svuint32_t svldff1_gather_[u32]offset[_u32](svbool_t pg, const uint32_t *base, svuint32_t offsets) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, Zoffsets.S, UXTW] + /// + public static unsafe Vector GatherVectorWithByteOffsetFirstFaulting(Vector mask, uint* address, Vector offsets) { throw new PlatformNotSupportedException(); } + + /// + /// svuint64_t svldff1_gather_[s64]offset[_u64](svbool_t pg, const uint64_t *base, svint64_t offsets) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, Zoffsets.D] + /// + public static unsafe Vector GatherVectorWithByteOffsetFirstFaulting(Vector mask, ulong* address, Vector offsets) { throw new PlatformNotSupportedException(); } + + /// + /// svuint64_t svldff1_gather_[u64]offset[_u64](svbool_t pg, const uint64_t *base, svuint64_t offsets) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, Zoffsets.D] + /// + public static unsafe Vector GatherVectorWithByteOffsetFirstFaulting(Vector mask, ulong* address, Vector offsets) { throw new PlatformNotSupportedException(); } + + + /// Unextended load /// /// svfloat64_t svld1_gather_[s64]offset[_f64](svbool_t pg, const float64_t *base, svint64_t offsets) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs index 44777bcb0123d..6a68727992cc1 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs @@ -4287,7 +4287,82 @@ internal Arm64() { } public static unsafe Vector GatherVectorUInt32ZeroExtend(Vector mask, uint* address, Vector indices) => GatherVectorUInt32ZeroExtend(mask, address, indices); - // Unextended load + /// Unextended load, first-faulting + + /// + /// svfloat64_t svldff1_gather_[s64]offset[_f64](svbool_t pg, const float64_t *base, svint64_t offsets) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, Zoffsets.D] + /// + public static unsafe Vector GatherVectorWithByteOffsetFirstFaulting(Vector mask, double* address, Vector offsets) => GatherVectorWithByteOffsetFirstFaulting(mask, address, offsets); + + /// + /// svfloat64_t svldff1_gather_[u64]offset[_f64](svbool_t pg, const float64_t *base, svuint64_t offsets) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, Zoffsets.D] + /// + public static unsafe Vector GatherVectorWithByteOffsetFirstFaulting(Vector mask, double* address, Vector offsets) => GatherVectorWithByteOffsetFirstFaulting(mask, address, offsets); + + /// + /// svint32_t svldff1_gather_[s32]offset[_s32](svbool_t pg, const int32_t *base, svint32_t offsets) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, Zoffsets.S, SXTW] + /// + public static unsafe Vector GatherVectorWithByteOffsetFirstFaulting(Vector mask, int* address, Vector offsets) => GatherVectorWithByteOffsetFirstFaulting(mask, address, offsets); + + /// + /// svint32_t svldff1_gather_[u32]offset[_s32](svbool_t pg, const int32_t *base, svuint32_t offsets) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, Zoffsets.S, UXTW] + /// + public static unsafe Vector GatherVectorWithByteOffsetFirstFaulting(Vector mask, int* address, Vector offsets) => GatherVectorWithByteOffsetFirstFaulting(mask, address, offsets); + + /// + /// svint64_t svldff1_gather_[s64]offset[_s64](svbool_t pg, const int64_t *base, svint64_t offsets) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, Zoffsets.D] + /// + public static unsafe Vector GatherVectorWithByteOffsetFirstFaulting(Vector mask, long* address, Vector offsets) => GatherVectorWithByteOffsetFirstFaulting(mask, address, offsets); + + /// + /// svint64_t svldff1_gather_[u64]offset[_s64](svbool_t pg, const int64_t *base, svuint64_t offsets) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, Zoffsets.D] + /// + public static unsafe Vector GatherVectorWithByteOffsetFirstFaulting(Vector mask, long* address, Vector offsets) => GatherVectorWithByteOffsetFirstFaulting(mask, address, offsets); + + /// + /// svfloat32_t svldff1_gather_[s32]offset[_f32](svbool_t pg, const float32_t *base, svint32_t offsets) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, Zoffsets.S, SXTW] + /// + public static unsafe Vector GatherVectorWithByteOffsetFirstFaulting(Vector mask, float* address, Vector offsets) => GatherVectorWithByteOffsetFirstFaulting(mask, address, offsets); + + /// + /// svfloat32_t svldff1_gather_[u32]offset[_f32](svbool_t pg, const float32_t *base, svuint32_t offsets) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, Zoffsets.S, UXTW] + /// + public static unsafe Vector GatherVectorWithByteOffsetFirstFaulting(Vector mask, float* address, Vector offsets) => GatherVectorWithByteOffsetFirstFaulting(mask, address, offsets); + + /// + /// svuint32_t svldff1_gather_[s32]offset[_u32](svbool_t pg, const uint32_t *base, svint32_t offsets) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, Zoffsets.S, SXTW] + /// + public static unsafe Vector GatherVectorWithByteOffsetFirstFaulting(Vector mask, uint* address, Vector offsets) => GatherVectorWithByteOffsetFirstFaulting(mask, address, offsets); + + /// + /// svuint32_t svldff1_gather_[u32]offset[_u32](svbool_t pg, const uint32_t *base, svuint32_t offsets) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, Zoffsets.S, UXTW] + /// + public static unsafe Vector GatherVectorWithByteOffsetFirstFaulting(Vector mask, uint* address, Vector offsets) => GatherVectorWithByteOffsetFirstFaulting(mask, address, offsets); + + /// + /// svuint64_t svldff1_gather_[s64]offset[_u64](svbool_t pg, const uint64_t *base, svint64_t offsets) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, Zoffsets.D] + /// + public static unsafe Vector GatherVectorWithByteOffsetFirstFaulting(Vector mask, ulong* address, Vector offsets) => GatherVectorWithByteOffsetFirstFaulting(mask, address, offsets); + + /// + /// svuint64_t svldff1_gather_[u64]offset[_u64](svbool_t pg, const uint64_t *base, svuint64_t offsets) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, Zoffsets.D] + /// + public static unsafe Vector GatherVectorWithByteOffsetFirstFaulting(Vector mask, ulong* address, Vector offsets) => GatherVectorWithByteOffsetFirstFaulting(mask, address, offsets); + + + /// Unextended load /// /// svfloat64_t svld1_gather_[s64]offset[_f64](svbool_t pg, const float64_t *base, svint64_t offsets) diff --git a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs index 90803aec54646..f4c916db234f7 100644 --- a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs +++ b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs @@ -4942,6 +4942,18 @@ internal Arm64() { } public static unsafe System.Numerics.Vector GatherVectorUInt32ZeroExtend(System.Numerics.Vector mask, uint* address, System.Numerics.Vector indices) { throw null; } public static System.Numerics.Vector GatherVectorUInt32ZeroExtend(System.Numerics.Vector mask, System.Numerics.Vector addresses) { throw null; } public static unsafe System.Numerics.Vector GatherVectorUInt32ZeroExtend(System.Numerics.Vector mask, uint* address, System.Numerics.Vector indices) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorWithByteOffsetFirstFaulting(System.Numerics.Vector mask, double* address, System.Numerics.Vector offsets) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorWithByteOffsetFirstFaulting(System.Numerics.Vector mask, double* address, System.Numerics.Vector offsets) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorWithByteOffsetFirstFaulting(System.Numerics.Vector mask, int* address, System.Numerics.Vector offsets) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorWithByteOffsetFirstFaulting(System.Numerics.Vector mask, int* address, System.Numerics.Vector offsets) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorWithByteOffsetFirstFaulting(System.Numerics.Vector mask, long* address, System.Numerics.Vector offsets) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorWithByteOffsetFirstFaulting(System.Numerics.Vector mask, long* address, System.Numerics.Vector offsets) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorWithByteOffsetFirstFaulting(System.Numerics.Vector mask, float* address, System.Numerics.Vector offsets) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorWithByteOffsetFirstFaulting(System.Numerics.Vector mask, float* address, System.Numerics.Vector offsets) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorWithByteOffsetFirstFaulting(System.Numerics.Vector mask, uint* address, System.Numerics.Vector offsets) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorWithByteOffsetFirstFaulting(System.Numerics.Vector mask, uint* address, System.Numerics.Vector offsets) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorWithByteOffsetFirstFaulting(System.Numerics.Vector mask, ulong* address, System.Numerics.Vector offsets) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorWithByteOffsetFirstFaulting(System.Numerics.Vector mask, ulong* address, System.Numerics.Vector offsets) { throw null; } public static unsafe System.Numerics.Vector GatherVectorWithByteOffsets(System.Numerics.Vector mask, double* address, System.Numerics.Vector offsets) { throw null; } public static unsafe System.Numerics.Vector GatherVectorWithByteOffsets(System.Numerics.Vector mask, double* address, System.Numerics.Vector offsets) { throw null; } public static unsafe System.Numerics.Vector GatherVectorWithByteOffsets(System.Numerics.Vector mask, int* address, System.Numerics.Vector offsets) { throw null; } diff --git a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs index d5b3d81b24623..e111a6b17cd72 100644 --- a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs +++ b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs @@ -3665,7 +3665,20 @@ ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_double_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "Double", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_long_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "Int64", ["GetFfrType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_ulong_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "UInt64", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), - + + ("SveGatherVectorByteOffsetFirstFaulting.template",new Dictionary { ["TestName"] = "Sve_GatherVectorWithByteOffsetFirstFaulting_float_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorWithByteOffsetFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["GetFfrType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()", ["ConvertFunc"] = "BitConverter.SingleToInt32Bits"}), + ("SveGatherVectorByteOffsetFirstFaulting.template",new Dictionary { ["TestName"] = "Sve_GatherVectorWithByteOffsetFirstFaulting_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorWithByteOffsetFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["GetFfrType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()", ["ConvertFunc"] = ""}), + ("SveGatherVectorByteOffsetFirstFaulting.template",new Dictionary { ["TestName"] = "Sve_GatherVectorWithByteOffsetFirstFaulting_uint_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorWithByteOffsetFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["GetFfrType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()", ["ConvertFunc"] = ""}), + ("SveGatherVectorByteOffsetFirstFaulting.template",new Dictionary { ["TestName"] = "Sve_GatherVectorWithByteOffsetFirstFaulting_float_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorWithByteOffsetFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["GetFfrType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()", ["ConvertFunc"] = "BitConverter.SingleToInt32Bits"}), + ("SveGatherVectorByteOffsetFirstFaulting.template",new Dictionary { ["TestName"] = "Sve_GatherVectorWithByteOffsetFirstFaulting_int_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorWithByteOffsetFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["GetFfrType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()", ["ConvertFunc"] = ""}), + ("SveGatherVectorByteOffsetFirstFaulting.template",new Dictionary { ["TestName"] = "Sve_GatherVectorWithByteOffsetFirstFaulting_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorWithByteOffsetFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["GetFfrType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()", ["ConvertFunc"] = ""}), + ("SveGatherVectorByteOffsetFirstFaulting.template",new Dictionary { ["TestName"] = "Sve_GatherVectorWithByteOffsetFirstFaulting_double_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorWithByteOffsetFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int64", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()", ["ConvertFunc"] = "BitConverter.DoubleToInt64Bits"}), + ("SveGatherVectorByteOffsetFirstFaulting.template",new Dictionary { ["TestName"] = "Sve_GatherVectorWithByteOffsetFirstFaulting_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorWithByteOffsetFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int64", ["GetFfrType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()", ["ConvertFunc"] = ""}), + ("SveGatherVectorByteOffsetFirstFaulting.template",new Dictionary { ["TestName"] = "Sve_GatherVectorWithByteOffsetFirstFaulting_ulong_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorWithByteOffsetFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int64", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()", ["ConvertFunc"] = ""}), + ("SveGatherVectorByteOffsetFirstFaulting.template",new Dictionary { ["TestName"] = "Sve_GatherVectorWithByteOffsetFirstFaulting_double_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorWithByteOffsetFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()", ["ConvertFunc"] = "BitConverter.DoubleToInt64Bits"}), + ("SveGatherVectorByteOffsetFirstFaulting.template",new Dictionary { ["TestName"] = "Sve_GatherVectorWithByteOffsetFirstFaulting_long_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorWithByteOffsetFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["GetFfrType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()", ["ConvertFunc"] = ""}), + ("SveGatherVectorByteOffsetFirstFaulting.template",new Dictionary { ["TestName"] = "Sve_GatherVectorWithByteOffsetFirstFaulting_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorWithByteOffsetFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()", ["ConvertFunc"] = ""}), + ("SveGatherVectorByteOffsets.template",new Dictionary {["TestName"] = "Sve_GatherVectorWithByteOffsets_float_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorWithByteOffsets", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()", ["ConvertFunc"] = "BitConverter.SingleToInt32Bits"}), ("SveGatherVectorByteOffsets.template",new Dictionary {["TestName"] = "Sve_GatherVectorWithByteOffsets_int_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorWithByteOffsets", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()", ["ConvertFunc"] = ""}), ("SveGatherVectorByteOffsets.template",new Dictionary {["TestName"] = "Sve_GatherVectorWithByteOffsets_uint_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorWithByteOffsets", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()", ["ConvertFunc"] = ""}), diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs index 9cbea6f844266..6fef8c2efb344 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs @@ -8571,6 +8571,47 @@ private static unsafe T GetGatherVectorBasesResultByIndex(data[index])); } + private static bool GetGatherVectorResultByByteOffset(int index, T[] mask, byte[] data, Offset[] offsets, T result) + where T : INumberBase + where Offset : IBinaryInteger + { + if (mask[index] == T.Zero) + { + return result == T.Zero; + } + + int offset = int.CreateChecked(offsets[index]); + + if (typeof(T) == typeof(int)) + { + return result == T.CreateTruncating(LoadInt32FromByteArray(data, offset)); + } + else if (typeof(T) == typeof(uint)) + { + return result == T.CreateTruncating(LoadUInt32FromByteArray(data, offset)); + } + else if (typeof(T) == typeof(long)) + { + return result == T.CreateTruncating(LoadInt64FromByteArray(data, offset)); + } + else if (typeof(T) == typeof(ulong)) + { + return result == T.CreateTruncating(LoadUInt64FromByteArray(data, offset)); + } + else if (typeof(T) == typeof(float)) + { + return BitConverter.SingleToInt32Bits((float)(object)result) == LoadInt32FromByteArray(data, offset); + } + else if (typeof(T) == typeof(double)) + { + return BitConverter.DoubleToInt64Bits((double)(object)result) == LoadInt64FromByteArray(data, offset); + } + else + { + return false; + } + } + private static bool CheckGatherVectorBehaviorCore(T[] mask, ExtendedElementT[] data, Index[] indices, T[] result, Func map) where T : INumberBase where ExtendedElementT : INumberBase @@ -8823,6 +8864,53 @@ public static bool CheckGatherVectorBasesFirstFaultingBehavior GetGatherVectorBasesResultByIndex(i, mask, data) == result[i]); } + public static bool CheckGatherVectorWithByteOffsetFirstFaultingBehavior(T[] mask, ExtendedElementT[] data, Offset[] offsets, T[] result, Vector faultResult) + where T : INumberBase + where ExtendedElementT : INumberBase + where Offset : IBinaryInteger + where TFault : INumberBase + { + // Checking first faulting behavior requires at least one zero to ensure we are testing the behavior. + if (!CheckFaultResultHasAtLeastOneZero(faultResult)) + { + TestLibrary.TestFramework.LogInformation("Fault result requires at least one zero."); + return false; + } + + var hasFaulted = false; + var expectedFaultResult = + InitVector(i => + { + if (hasFaulted) + { + return TFault.Zero; + } + + if (mask[i] == T.Zero) + { + return TFault.One; + } + + var offset = int.CreateChecked(offsets[i]); + var endOffset = data.Length * Unsafe.SizeOf(); + if (offset < 0 || offset >= endOffset) + { + hasFaulted = true; + return TFault.Zero; + } + return TFault.One; + }); + if (expectedFaultResult != faultResult) + { + TestLibrary.TestFramework.LogInformation($"Expected fault result: {expectedFaultResult}\nActual fault result: {faultResult}"); + return false; + } + + byte[] bytes = new byte[data.Length * Unsafe.SizeOf()]; + Buffer.BlockCopy(data, 0, bytes, 0, bytes.Length); + return CheckFirstFaultingBehaviorCore(result, faultResult, i => GetGatherVectorResultByByteOffset(i, mask, bytes, offsets, result[i])); + } + public static T[] CreateBreakPropagateMask(T[] op1, T[] op2) where T : IBinaryInteger { var count = op1.Length; diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorByteOffsetFirstFaulting.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorByteOffsetFirstFaulting.template new file mode 100644 index 0000000000000..6cb1706098a50 --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorByteOffsetFirstFaulting.template @@ -0,0 +1,764 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +/****************************************************************************** + * This file is auto-generated from a template file by the GenerateTests.csx * + * script in tests\src\JIT\HardwareIntrinsics\Arm\Shared. In order to make * + * changes, please update the corresponding template and run according to the * + * directions listed in the file. * + ******************************************************************************/ + +using System; +using System.Buffers; +using System.Numerics; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using Xunit; + +namespace JIT.HardwareIntrinsics.Arm +{ + public static partial class Program + { + [Fact] + public static void {TestName}() + { + var test = new SveGatherVectorIndices__{TestName}(); + + if (test.IsSupported) + { + // Validates basic functionality works, using Unsafe.Read + test.RunBasicScenario_UnsafeRead(); + + // Validates basic functionality works + test.RunBasicScenario_Load(); + + // Validates basic functionality of first-faulting behavior + test.RunBasicScenario_LoadFirstFaulting(); + + // Validates fully masked out load works. + test.RunBasicScenario_FalseMask(); + + // Validates fully masked out load with invalid address works. + test.RunBasicScenario_NonFaulting(); + + // Validates calling via reflection works, using Unsafe.Read + test.RunReflectionScenario_UnsafeRead(); + + // Validates passing a local works, using Unsafe.Read + test.RunLclVarScenario_UnsafeRead(); + + // Validates passing an instance member of a class works + test.RunClassFldScenario(); + + // Validates passing the field of a local struct works + test.RunStructLclFldScenario(); + + // Validates passing an instance member of a struct works + test.RunStructFldScenario(); + + // Validates using inside ConditionalSelect with value falseValue + // Currently, using this operation in ConditionalSelect() gives incorrect result + // when falseReg == targetReg because this instruction uses Pg/Z to update the targetReg + // instead of Pg/M to merge it. As such, the value of falseReg is lost. Ideally, such + // instructions should be marked similar to RMW (a different flag name) to make sure that + // we do not assign falseReg/targetReg same. Then, we would do something like this: + // + // ldnf1sh target, pg/z, [x0] + // sel mask, target, target, falseReg + // + // This needs more careful thinking, so disabling it for now. + // test.ConditionalSelect_FalseOp(); + + // Validates using inside ConditionalSelect with zero falseValue + test.ConditionalSelect_ZeroOp(); + } + else + { + // Validates we throw on unsupported hardware + test.RunUnsupportedScenario(); + } + + if (!test.Succeeded) + { + throw new Exception("One or more scenarios did not complete as expected."); + } + } + } + + public sealed unsafe class SveGatherVectorIndices__{TestName} + { + private struct DataTable + { + private byte[] inArray1; + private byte[] inArray2; + private byte[] inArray3; + private byte[] outArray; + + private GCHandle inHandle1; + private GCHandle inHandle2; + private GCHandle inHandle3; + private GCHandle outHandle; + + private ulong alignment; + + public DataTable({Op1BaseType}[] inArray1, {Op2BaseType}[] inArray2, {Op3BaseType}[] inArray3, {RetBaseType}[] outArray, int alignment) + { + int sizeOfinArray1 = inArray1.Length * Unsafe.SizeOf<{Op1BaseType}>(); + int sizeOfinArray2 = inArray2.Length * Unsafe.SizeOf<{Op2BaseType}>(); + int sizeOfinArray3 = inArray3.Length * Unsafe.SizeOf<{Op3BaseType}>(); + int sizeOfinBounded = new Random().Next(Unsafe.SizeOf<{Op2BaseType}>(), Vector<{Op2BaseType}>.Count * Unsafe.SizeOf<{Op2BaseType}>() - 1); + int sizeOfoutArray = outArray.Length * Unsafe.SizeOf<{RetBaseType}>(); + + if ((alignment != 64 && alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfinArray1 || (alignment * 2) < sizeOfinArray3 || (alignment * 2) < sizeOfoutArray) + { + throw new ArgumentException($"Invalid value of alignment: {alignment}, sizeOfinArray1: {sizeOfinArray1}, sizeOfinArray2: {sizeOfinArray2}, sizeOfinArray3: {sizeOfinArray3}, sizeOfoutArray: {sizeOfoutArray}"); + } + + this.inArray1 = new byte[alignment * 2]; + this.inArray2 = new byte[alignment + sizeOfinArray2]; + this.inArray3 = new byte[alignment * 2]; + this.outArray = new byte[alignment * 2]; + + this.inHandle1 = GCHandle.Alloc(this.inArray1, GCHandleType.Pinned); + this.inHandle2 = GCHandle.Alloc(this.inArray2, GCHandleType.Pinned); + this.inHandle3 = GCHandle.Alloc(this.inArray3, GCHandleType.Pinned); + this.outHandle = GCHandle.Alloc(this.outArray, GCHandleType.Pinned); + + this.alignment = (ulong)alignment; + + this.inBounded = BoundedMemory.Allocate(sizeOfinBounded, PoisonPagePlacement.After); + + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray1Ptr), ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), (uint)sizeOfinArray1); + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray2Ptr), ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), (uint)sizeOfinArray2); + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray3Ptr), ref Unsafe.As<{Op3BaseType}, byte>(ref inArray3[0]), (uint)sizeOfinArray3); + Unsafe.CopyBlockUnaligned(ref inBounded.Span.GetPinnableReference(), ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), (uint)sizeOfinBounded); + } + + public BoundedMemory inBounded; + + public void* inArray1Ptr => Align((byte*)(inHandle1.AddrOfPinnedObject().ToPointer()), alignment); + public void* inArray2Ptr => Align((byte*)(inHandle2.AddrOfPinnedObject().ToPointer()), alignment); + public void* inArray3Ptr => Align((byte*)(inHandle3.AddrOfPinnedObject().ToPointer()), alignment); + public void* outArrayPtr => Align((byte*)(outHandle.AddrOfPinnedObject().ToPointer()), alignment); + + public void Dispose() + { + inHandle1.Free(); + inHandle2.Free(); + inHandle3.Free(); + outHandle.Free(); + } + + private static unsafe void* Align(byte* buffer, ulong expectedAlignment) + { + return (void*)(((ulong)buffer + expectedAlignment - 1) & ~(expectedAlignment - 1)); + } + } + + private struct TestStruct + { + public {Op1VectorType}<{Op1BaseType}> _fld1; + public {Op2BaseType}* _fld2; + public {Op3VectorType}<{Op3BaseType}> _fld3; + + public static TestStruct Create(DataTable _dataTable) + { + var testStruct = new TestStruct(); + + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + for (var i = 0; i < Op2ElementCount; i++) { _data2[i] = {NextValueOp2}; } + + // Ensure all values of _data3 fit within the number of _data2 elements + for (var i = 0; i < Op3ElementCount; i++) { _data3[i] = {NextValueOp3} % ({Op3BaseType})Op2ElementCount; } + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref testStruct._fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op3VectorType}<{Op3BaseType}>, byte>(ref testStruct._fld3), ref Unsafe.As<{Op3BaseType}, byte>(ref _data3[0]), (uint)Unsafe.SizeOf<{Op3VectorType}<{Op3BaseType}>>()); + + testStruct._fld2 = ({Op2BaseType}*)_dataTable.inArray2Ptr; + + return testStruct; + } + + public void RunStructFldScenario(SveGatherVectorIndices__{TestName} testClass) + { + var result = {Isa}.{Method}(_fld1, _fld2, _fld3); + + Unsafe.Write(testClass._dataTable.outArrayPtr, result); + testClass.ValidateResult(_fld1, _fld2, _fld3, testClass._dataTable.outArrayPtr); + } + } + + private static readonly int LargestVectorSize = {LargestVectorSize}; + + // A large enough buffer to hold many values. Op3 will index into Op2. + private static readonly int Op2ElementCount = 1024; + + private static readonly int Op1ElementCount = Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>() / sizeof({Op1BaseType}); + private static readonly int Op3ElementCount = Unsafe.SizeOf<{Op3VectorType}<{Op3BaseType}>>() / sizeof({Op3BaseType}); + private static readonly int RetElementCount = Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>() / sizeof({RetBaseType}); + + private static {Op1BaseType}[] _data1 = new {Op1BaseType}[Op1ElementCount]; + private static {Op2BaseType}[] _data2 = new {Op2BaseType}[Op2ElementCount]; + private static {Op3BaseType}[] _data3 = new {Op3BaseType}[Op3ElementCount]; + + private static {RetBaseType}[] _maskData = new {RetBaseType}[RetElementCount]; + private static {RetBaseType}[] _falseData = new {RetBaseType}[RetElementCount]; + + private {Op1VectorType}<{Op1BaseType}> _fld1; + private {Op2BaseType}* _fld2; + private {Op3VectorType}<{Op3BaseType}> _fld3; + + private {Op1VectorType}<{Op1BaseType}> _mask; + private {Op1VectorType}<{Op1BaseType}> _falseFld; + + private DataTable _dataTable; + + public SveGatherVectorIndices__{TestName}() + { + Succeeded = true; + + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + for (var i = 0; i < Op2ElementCount; i++) { _data2[i] = {NextValueOp2}; } + + // Ensure all values of _data3 fit within the number of _data2 elements + for (var i = 0; i < Op3ElementCount; i++) { _data3[i] = {NextValueOp3} % ({Op3BaseType})Op2ElementCount; } + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op3VectorType}<{Op3BaseType}>, byte>(ref _fld3), ref Unsafe.As<{Op3BaseType}, byte>(ref _data3[0]), (uint)Unsafe.SizeOf<{Op3VectorType}<{Op3BaseType}>>()); + + for (var i = 0; i < RetElementCount; i++) { _maskData[i] = ({RetBaseType})({NextValueOp1}); } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetVectorType}<{RetBaseType}>, byte>(ref _mask), ref Unsafe.As<{RetBaseType}, byte>(ref _maskData[0]), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + for (var i = 0; i < RetElementCount; i++) { _falseData[i] = ({RetBaseType})({NextValueOp2}); } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetVectorType}<{RetBaseType}>, byte>(ref _falseFld), ref Unsafe.As<{RetBaseType}, byte>(ref _falseData[0]), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + _dataTable = new DataTable(_data1, _data2, _data3, new {RetBaseType}[RetElementCount], LargestVectorSize); + + _fld2 = ({Op2BaseType}*)_dataTable.inArray2Ptr; + } + + public bool IsSupported => {Isa}.IsSupported; + + public bool Succeeded { get; set; } + + public void RunBasicScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_UnsafeRead)); + + var result = {Isa}.{Method}( + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr), + ({Op2BaseType}*)_dataTable.inArray2Ptr, + Unsafe.Read<{Op3VectorType}<{Op3BaseType}>>(_dataTable.inArray3Ptr) + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.inArray3Ptr, _dataTable.outArrayPtr); + } + + public void RunBasicScenario_Load() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_Load)); + + {Op1VectorType}<{Op1BaseType}> loadMask1 = Sve.CreateTrueMask{Op1BaseType}(SveMaskPattern.All); + {Op3VectorType}<{Op3BaseType}> loadMask3 = Sve.CreateTrueMask{Op3BaseType}(SveMaskPattern.All); + + var result = {Isa}.{Method}( + {LoadIsa}.Load{Op1VectorType}(loadMask1, ({Op1BaseType}*)(_dataTable.inArray1Ptr)), + ({Op2BaseType}*)_dataTable.inArray2Ptr, + {LoadIsa}.Load{Op3VectorType}(loadMask3, ({Op3BaseType}*)(_dataTable.inArray3Ptr)) + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.inArray3Ptr, _dataTable.outArrayPtr); + } + + public void RunBasicScenario_LoadFirstFaulting() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_LoadFirstFaulting)); + + {Op1VectorType}<{Op1BaseType}> loadMask1 = Sve.CreateTrueMask{Op1BaseType}(SveMaskPattern.All); + {Op3VectorType}<{Op3BaseType}> loadMask3 = Sve.CreateTrueMask{Op3BaseType}(SveMaskPattern.All); + + var op1 = {LoadIsa}.Load{Op1VectorType}(loadMask1, ({Op1BaseType}*)(_dataTable.inArray1Ptr)); + ref var op2Ref = ref (_dataTable.inBounded.Span.GetPinnableReference()); + var op3 = {LoadIsa}.Load{Op3VectorType}(loadMask3, ({Op3BaseType}*)(_dataTable.inArray3Ptr)); + + // We know this is outside the bounds because 'inBounded' will never be the full size of a Vector. + var outsideBoundsIndex = ({Op3BaseType})(Vector<{Op2BaseType}>.Count - 1); + + // When testing first-faulting behavior, we need to make sure we can get the first element. + // So set the first active element of the index vector to 0. + var firstActiveElement = -1; + for (var i = 0; i < Vector<{Op3BaseType}>.Count; i++) + { + // op1 is the mask for GatherVector. + if (op1[i] != 0) + { + if (firstActiveElement == -1) + { + op3 = op3.WithElement<{Op3BaseType}>(i, 0); + firstActiveElement = i; + } + else if (op3[i] < 0 || op3[i] > (outsideBoundsIndex * sizeof({Op2BaseType}))) + { + op3 = op3.WithElement<{Op3BaseType}>(i, (outsideBoundsIndex * sizeof({Op2BaseType}))); + } + } + } + + // Force at least one element to cause a fault (required for testing). So set the last element to an index outside the bounds. + var lastIndex = Vector<{Op3BaseType}>.Count - 1; + + // Ensure we at least have one element that we can read. + if ((firstActiveElement == -1) || (firstActiveElement == lastIndex)) + { + op1 = op1.WithElement<{Op1BaseType}>(0, 1); + op3 = op3.WithElement<{Op3BaseType}>(0, 0); + } + + op1 = op1.WithElement<{Op1BaseType}>(lastIndex, 1); + + // Force an index outside the bounds. + op3 = op3.WithElement<{Op3BaseType}>(lastIndex, (outsideBoundsIndex * sizeof({Op2BaseType}))); + + Sve.SetFfr(Sve.CreateTrueMaskByte(SveMaskPattern.All)); + var result = {Isa}.{Method}( + op1, + ({Op2BaseType}*)(Unsafe.AsPointer(ref op2Ref)), + op3 + ); + var faultResult = Sve.GetFfr{GetFfrType}(); + + ref var op1Ref = ref op1; + ref var op3Ref = ref op3; + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateFirstFaultingResult(Unsafe.AsPointer(ref op1Ref), ref op2Ref, _dataTable.inBounded.Span.Length, Unsafe.AsPointer(ref op3Ref), _dataTable.outArrayPtr, faultResult); + } + + public void RunBasicScenario_FalseMask() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_FalseMask)); + + {Op1VectorType}<{Op1BaseType}> falseMask = Sve.CreateFalseMask{Op1BaseType}(); + + var result = {Isa}.{Method}( + falseMask, + ({Op2BaseType}*)_dataTable.inArray2Ptr, + Unsafe.Read<{Op3VectorType}<{Op3BaseType}>>(_dataTable.inArray3Ptr) + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateZeroResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.inArray3Ptr, _dataTable.outArrayPtr); + } + + public void RunBasicScenario_NonFaulting() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_NonFaulting)); + + {Op1VectorType}<{Op1BaseType}> falseMask = Sve.CreateFalseMask{Op1BaseType}(); + + try + { + var result = {Isa}.{Method}( + falseMask, + default, + Unsafe.Read<{Op3VectorType}<{Op3BaseType}>>(_dataTable.inArray3Ptr) + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateZeroResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.inArray3Ptr, _dataTable.outArrayPtr); + } + catch + { + Succeeded = false; + } + } + + public void RunReflectionScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunReflectionScenario_UnsafeRead)); + + var result = typeof({Isa}).GetMethod(nameof({Isa}.{Method}), new Type[] { typeof({Op1VectorType}<{Op1BaseType}>), typeof({Op2BaseType}*), typeof({Op3VectorType}<{Op3BaseType}>) }) + .Invoke(null, new object[] { + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr), + Pointer.Box(_dataTable.inArray2Ptr, typeof({Op2BaseType}*)), + Unsafe.Read<{Op3VectorType}<{Op3BaseType}>>(_dataTable.inArray3Ptr) + }); + + Unsafe.Write(_dataTable.outArrayPtr, ({RetVectorType}<{RetBaseType}>)(result)); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.inArray3Ptr, _dataTable.outArrayPtr); + } + + public void RunLclVarScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunLclVarScenario_UnsafeRead)); + + var op1 = Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr); + var op2 = ({Op2BaseType}*)_dataTable.inArray2Ptr; + var op3 = Unsafe.Read<{Op3VectorType}<{Op3BaseType}>>(_dataTable.inArray3Ptr); + var result = {Isa}.{Method}(op1, op2, op3); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(op1, op2, op3, _dataTable.outArrayPtr); + } + + public void RunClassFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunClassFldScenario)); + + var result = {Isa}.{Method}(_fld1, _fld2, _fld3); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_fld1, _fld2, _fld3, _dataTable.outArrayPtr); + } + + public void RunStructLclFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructLclFldScenario)); + + var test = TestStruct.Create(_dataTable); + var result = {Isa}.{Method}(test._fld1, test._fld2, test._fld3); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(test._fld1, test._fld2, test._fld3, _dataTable.outArrayPtr); + } + + public void RunStructFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructFldScenario)); + + var test = TestStruct.Create(_dataTable); + test.RunStructFldScenario(this); + } + + public void ConditionalSelect_FalseOp() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_mask - operation in TrueValue"); + ConditionalSelectScenario_TrueValue(_mask, _fld1, _fld2, _fld3, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_zero - operation in TrueValue"); + ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld3, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_all - operation in TrueValue"); + ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld3, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_mask - operation in FalseValue"); + ConditionalSelectScenario_FalseValue(_mask, _fld1, _fld2, _fld3, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_zero - operation in FalseValue"); + ConditionalSelectScenario_FalseValue({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld3, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_all - operation in FalseValue"); + ConditionalSelectScenario_FalseValue({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld3, _falseFld); + } + + public void ConditionalSelect_ZeroOp() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_mask - operation in TrueValue"); + ConditionalSelectScenario_TrueValue(_mask, _fld1, _fld2, _fld3, {Op1VectorType}<{RetBaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero - operation in TrueValue"); + ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld3, {Op1VectorType}<{Op1BaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all - operation in TrueValue"); + ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld3, {Op1VectorType}<{Op1BaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_mask - operation in FalseValue"); + ConditionalSelectScenario_FalseValue(_mask, _fld1, _fld2, _fld3, {Op1VectorType}<{RetBaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero - operation in FalseValue"); + ConditionalSelectScenario_FalseValue({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld3, {Op1VectorType}<{Op1BaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all - operation in FalseValue"); + ConditionalSelectScenario_FalseValue({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld3, {Op1VectorType}<{Op1BaseType}>.Zero); + } + + [method: MethodImpl(MethodImplOptions.AggressiveInlining)] + private void ConditionalSelectScenario_TrueValue({RetVectorType}<{RetBaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op2BaseType}* op2, {Op3VectorType}<{Op3BaseType}> op3, {Op1VectorType}<{Op1BaseType}> falseOp) + { + var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3), falseOp); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateConditionalSelectResult_TrueValue(mask, op1, op2, op3, falseOp, _dataTable.outArrayPtr); + } + + [method: MethodImpl(MethodImplOptions.AggressiveInlining)] + private void ConditionalSelectScenario_FalseValue({RetVectorType}<{RetBaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op2BaseType}* op2, {Op3VectorType}<{Op3BaseType}> op3, {Op1VectorType}<{Op1BaseType}> trueOp) + { + var result = Sve.ConditionalSelect(mask, trueOp, {Isa}.{Method}(op1, op2, op3)); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateConditionalSelectResult_FalseValue(mask, op1, op2, op3, trueOp, _dataTable.outArrayPtr); + } + + public void RunUnsupportedScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunUnsupportedScenario)); + + Succeeded = false; + + try + { + RunBasicScenario_Load(); + } + catch (PlatformNotSupportedException) + { + Succeeded = true; + } + } + + private void ValidateResult({Op1VectorType}<{Op1BaseType}> op1, {Op2BaseType}* op2, {Op3VectorType}<{Op3BaseType}> op3, void* result, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + byte[] inArray2 = new byte[Unsafe.SizeOf<{Op2BaseType}>() * Op2ElementCount]; + {Op3BaseType}[] inArray3 = new {Op3BaseType}[Op3ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), op1); + Unsafe.CopyBlockUnaligned(ref Unsafe.As(ref inArray2[0]), ref Unsafe.AsRef(op2), (uint)(Unsafe.SizeOf<{Op2BaseType}>() * Op2ElementCount)); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op3BaseType}, byte>(ref inArray3[0]), op3); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateResult(inArray1, inArray2, inArray3, outArray, method); + } + + private void ValidateResult(void* firstOp, void* secondOp, void* thirdOp, void* result, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + byte[] inArray2 = new byte[Unsafe.SizeOf<{Op2BaseType}>() * Op2ElementCount]; + {Op3BaseType}[] inArray3 = new {Op3BaseType}[Op3ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), ref Unsafe.AsRef(firstOp), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As(ref inArray2[0]), ref Unsafe.AsRef(secondOp), (uint)(Unsafe.SizeOf<{Op2BaseType}>() * Op2ElementCount)); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op3BaseType}, byte>(ref inArray3[0]), ref Unsafe.AsRef(thirdOp), (uint)Unsafe.SizeOf<{Op3VectorType}<{Op3BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateResult(inArray1, inArray2, inArray3, outArray, method); + } + + private void ValidateResult({Op1BaseType}[] firstOp, byte[] secondOp, {Op3BaseType}[] thirdOp, {RetBaseType}[] result, [CallerMemberName] string method = "") + { + bool succeeded = true; + + for (var i = 0; i < RetElementCount; i++) + { + {RetBaseType} element = Helpers.Load{RetBaseType}FromByteArray(secondOp, thirdOp[i]); + {RetBaseType} gatherResult = ({RetBaseType})(firstOp[i] == 0 ? 0 : element); + if ({ConvertFunc}(result[i]) != {ConvertFunc}(gatherResult)) + { + succeeded = false; + break; + } + } + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1BaseType}, {Op2BaseType}, {Op3BaseType}): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})"); + TestLibrary.TestFramework.LogInformation($" secondOp: ({string.Join(", ", secondOp)})"); + TestLibrary.TestFramework.LogInformation($" thirdOp: ({string.Join(", ", thirdOp)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + + private void ValidateZeroResult({Op1VectorType}<{Op1BaseType}> op1, {Op2BaseType}* op2, {Op3VectorType}<{Op3BaseType}> op3, void* result, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + byte[] inArray2 = new byte[Unsafe.SizeOf<{Op2BaseType}>() * Op2ElementCount]; + {Op3BaseType}[] inArray3 = new {Op3BaseType}[Op3ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), op1); + Unsafe.CopyBlockUnaligned(ref Unsafe.As(ref inArray2[0]), ref Unsafe.AsRef(op2), (uint)(Unsafe.SizeOf<{Op2BaseType}>() * Op2ElementCount)); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op3BaseType}, byte>(ref inArray3[0]), op3); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateZeroResult(inArray1, inArray2, inArray3, outArray, method); + } + + private void ValidateZeroResult(void* firstOp, void* secondOp, void* thirdOp, void* result, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + byte[] inArray2 = new byte[Unsafe.SizeOf<{Op2BaseType}>() * Op2ElementCount]; + {Op3BaseType}[] inArray3 = new {Op3BaseType}[Op3ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), ref Unsafe.AsRef(firstOp), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As(ref inArray2[0]), ref Unsafe.AsRef(secondOp), (uint)(Unsafe.SizeOf<{Op2BaseType}>() * Op2ElementCount)); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op3BaseType}, byte>(ref inArray3[0]), ref Unsafe.AsRef(thirdOp), (uint)Unsafe.SizeOf<{Op3VectorType}<{Op3BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateZeroResult(inArray1, inArray2, inArray3, outArray, method); + } + + private void ValidateZeroResult({Op1BaseType}[] firstOp, byte[] secondOp, {Op3BaseType}[] thirdOp, {RetBaseType}[] result, [CallerMemberName] string method = "") + { + bool succeeded = true; + + for (var i = 0; i < RetElementCount; i++) + { + if (result[i] != 0) + { + succeeded = false; + break; + } + } + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1BaseType}, {Op2BaseType}, {Op3BaseType}): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})"); + TestLibrary.TestFramework.LogInformation($" secondOp: ({string.Join(", ", secondOp)})"); + TestLibrary.TestFramework.LogInformation($" thirdOp: ({string.Join(", ", thirdOp)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + + private void ValidateConditionalSelectResult_TrueValue({RetVectorType}<{RetBaseType}> maskOp, {Op1VectorType}<{Op1BaseType}> op1, {Op2BaseType}* op2, {Op3VectorType}<{Op3BaseType}> op3, {RetVectorType}<{RetBaseType}> falseOp, void* result, [CallerMemberName] string method = "") + { + {RetBaseType}[] maskArray = new {RetBaseType}[RetElementCount]; + {Op1BaseType}[] op1Array = new {Op1BaseType}[Op1ElementCount]; + byte[] op2Array = new byte[Unsafe.SizeOf<{Op2BaseType}>() * Op2ElementCount]; + {Op3BaseType}[] op3Array = new {Op3BaseType}[Op3ElementCount]; + {RetBaseType}[] falseValArray = new {RetBaseType}[RetElementCount]; + {RetBaseType}[] resultArray = new {RetBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref maskArray[0]), maskOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref op1Array[0]), op1); + Unsafe.CopyBlockUnaligned(ref Unsafe.As(ref op2Array[0]), ref Unsafe.AsRef(op2), (uint)(Unsafe.SizeOf<{Op2BaseType}>() * Op2ElementCount)); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op3BaseType}, byte>(ref op3Array[0]), op3); + Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref falseValArray[0]), falseOp); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref resultArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateConditionalSelectResult_TrueValue(maskArray, op1Array, op2Array, op3Array, falseValArray, resultArray, method); + } + + private void ValidateConditionalSelectResult_TrueValue({RetBaseType}[] maskOp, {Op1BaseType}[] firstOp, byte[] secondOp, {Op3BaseType}[] thirdOp, {RetBaseType}[] falseOp, {RetBaseType}[] result, [CallerMemberName] string method = "") + { + bool succeeded = true; + + for (var i = 0; i < RetElementCount; i++) + { + {RetBaseType} element = Helpers.Load{RetBaseType}FromByteArray(secondOp, thirdOp[i]); + {RetBaseType} gatherResult = ({RetBaseType})(firstOp[i] == 0 ? 0 : element); + {RetBaseType} iterResult = (maskOp[i] != 0) ? gatherResult : falseOp[i]; + if ({ConvertFunc}(iterResult) != {ConvertFunc}(result[i])) + { + succeeded = false; + break; + } + } + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1BaseType}, {Op2BaseType}, {Op3BaseType}): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" maskOp: ({string.Join(", ", maskOp)})"); + TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})"); + TestLibrary.TestFramework.LogInformation($" secondOp: ({string.Join(", ", secondOp)})"); + TestLibrary.TestFramework.LogInformation($" thirdOp: ({string.Join(", ", thirdOp)})"); + TestLibrary.TestFramework.LogInformation($" falseOp: ({string.Join(", ", falseOp)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + + private void ValidateConditionalSelectResult_FalseValue({RetVectorType}<{RetBaseType}> maskOp, {Op1VectorType}<{Op1BaseType}> op1, {Op2BaseType}* op2, {Op3VectorType}<{Op3BaseType}> op3, {RetVectorType}<{RetBaseType}> trueOp, void* result, [CallerMemberName] string method = "") + { + {RetBaseType}[] maskArray = new {RetBaseType}[RetElementCount]; + {Op1BaseType}[] op1Array = new {Op1BaseType}[Op1ElementCount]; + byte[] op2Array = new byte[Unsafe.SizeOf<{Op2BaseType}>() * Op2ElementCount]; + {Op3BaseType}[] op3Array = new {Op3BaseType}[Op3ElementCount]; + {RetBaseType}[] trueValArray = new {RetBaseType}[RetElementCount]; + {RetBaseType}[] resultArray = new {RetBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref maskArray[0]), maskOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref op1Array[0]), op1); + Unsafe.CopyBlockUnaligned(ref Unsafe.As(ref op2Array[0]), ref Unsafe.AsRef(op2), (uint)(Unsafe.SizeOf<{Op2BaseType}>() * Op2ElementCount)); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op3BaseType}, byte>(ref op3Array[0]), op3); + Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref trueValArray[0]), trueOp); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref resultArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateConditionalSelectResult_FalseValue(maskArray, op1Array, op2Array, op3Array, trueValArray, resultArray, method); + } + + private void ValidateConditionalSelectResult_FalseValue({RetBaseType}[] maskOp, {Op1BaseType}[] firstOp, byte[] secondOp, {Op3BaseType}[] thirdOp, {RetBaseType}[] trueOp, {RetBaseType}[] result, [CallerMemberName] string method = "") + { + bool succeeded = true; + + for (var i = 0; i < RetElementCount; i++) + { + {RetBaseType} element = Helpers.Load{RetBaseType}FromByteArray(secondOp, thirdOp[i]); + {RetBaseType} gatherResult = ({RetBaseType})(firstOp[i] == 0 ? 0 : element); + {RetBaseType} iterResult = (maskOp[i] != 0) ? trueOp[i] : gatherResult; + if (maskOp[i] != 0) + { + if ({ConvertFunc}(iterResult) != {ConvertFunc}(result[i])) + { + succeeded = false; + break; + } + } + } + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1BaseType}, {Op2BaseType}, {Op3BaseType}): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" maskOp: ({string.Join(", ", maskOp)})"); + TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})"); + TestLibrary.TestFramework.LogInformation($" secondOp: ({string.Join(", ", secondOp)})"); + TestLibrary.TestFramework.LogInformation($" thirdOp: ({string.Join(", ", thirdOp)})"); + TestLibrary.TestFramework.LogInformation($" trueOp: ({string.Join(", ", trueOp)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + + private void ValidateFirstFaultingResult(void* op1, ref byte op2, int op2Size, void* op3, void* result, Vector<{GetFfrType}> faultResult, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Vector<{Op1BaseType}>.Count]; + {Op2BaseType}[] inArray2 = new {Op2BaseType}[op2Size / Unsafe.SizeOf<{Op2BaseType}>()]; + {Op3BaseType}[] inArray3 = new {Op3BaseType}[Vector<{Op3BaseType}>.Count]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), ref Unsafe.AsRef(op1), (uint)(inArray1.Length * Unsafe.SizeOf<{Op1BaseType}>())); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), ref op2, (uint)(inArray2.Length * Unsafe.SizeOf<{Op2BaseType}>())); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op3BaseType}, byte>(ref inArray3[0]), ref Unsafe.AsRef(op3), (uint)(inArray3.Length * Unsafe.SizeOf<{Op3BaseType}>())); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateFirstFaultingResult(inArray1, inArray2, inArray3, outArray, faultResult, method); + } + + private void ValidateFirstFaultingResult({Op1BaseType}[] firstOp, {Op2BaseType}[] secondOp, {Op3BaseType}[] thirdOp, {RetBaseType}[] result, Vector<{GetFfrType}> faultResult, [CallerMemberName] string method = "") + { + var succeeded = Helpers.CheckGatherVectorWithByteOffsetFirstFaultingBehavior(firstOp, secondOp, thirdOp, result, faultResult); + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1BaseType}, {Op2BaseType}, {Op3BaseType}): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})"); + TestLibrary.TestFramework.LogInformation($" secondOp: ({string.Join(", ", secondOp)})"); + TestLibrary.TestFramework.LogInformation($" thirdOp: ({string.Join(", ", thirdOp)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation($" faultResult: ({faultResult})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + } +} \ No newline at end of file