diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 6ad76fcd48cd5..f5b5c1e853545 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -677,10 +677,47 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { if (!instrIsRMW) { - // Perform the actual "predicated" operation so that `embMaskOp1Reg` is the first operand - // and `embMaskOp2Reg` is the second operand. - GetEmitter()->emitIns_R_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, - embMaskOp2Reg, opt); + // Perform the actual "predicated" operation so that `embMaskOp1Reg` is the first operand.. + switch (intrinEmbMask.id) + { + case NI_Sve_LoadVectorByteNonFaultingZeroExtendToInt16: + case NI_Sve_LoadVectorByteNonFaultingZeroExtendToInt32: + case NI_Sve_LoadVectorByteNonFaultingZeroExtendToInt64: + case NI_Sve_LoadVectorByteNonFaultingZeroExtendToUInt16: + case NI_Sve_LoadVectorByteNonFaultingZeroExtendToUInt32: + case NI_Sve_LoadVectorByteNonFaultingZeroExtendToUInt64: + case NI_Sve_LoadVectorInt16NonFaultingSignExtendToInt32: + case NI_Sve_LoadVectorInt16NonFaultingSignExtendToInt64: + case NI_Sve_LoadVectorInt16NonFaultingSignExtendToUInt32: + case NI_Sve_LoadVectorInt16NonFaultingSignExtendToUInt64: + case NI_Sve_LoadVectorInt32NonFaultingSignExtendToInt64: + case NI_Sve_LoadVectorInt32NonFaultingSignExtendToUInt64: + case NI_Sve_LoadVectorNonFaulting: + case NI_Sve_LoadVectorSByteNonFaultingSignExtendToInt16: + case NI_Sve_LoadVectorSByteNonFaultingSignExtendToInt32: + case NI_Sve_LoadVectorSByteNonFaultingSignExtendToInt64: + case NI_Sve_LoadVectorSByteNonFaultingSignExtendToUInt16: + case NI_Sve_LoadVectorSByteNonFaultingSignExtendToUInt32: + case NI_Sve_LoadVectorSByteNonFaultingSignExtendToUInt64: + case NI_Sve_LoadVectorUInt16NonFaultingZeroExtendToInt32: + case NI_Sve_LoadVectorUInt16NonFaultingZeroExtendToInt64: + case NI_Sve_LoadVectorUInt16NonFaultingZeroExtendToUInt32: + case NI_Sve_LoadVectorUInt16NonFaultingZeroExtendToUInt64: + case NI_Sve_LoadVectorUInt32NonFaultingZeroExtendToInt64: + case NI_Sve_LoadVectorUInt32NonFaultingZeroExtendToUInt64: + { + + GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, + opt); + break; + } + + default: + { + GetEmitter()->emitIns_R_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, + embMaskOp2Reg, opt); + } + } break; } @@ -2089,6 +2126,43 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) break; } + case NI_Sve_LoadVectorByteNonFaultingZeroExtendToInt16: + case NI_Sve_LoadVectorByteNonFaultingZeroExtendToInt32: + case NI_Sve_LoadVectorByteNonFaultingZeroExtendToInt64: + case NI_Sve_LoadVectorByteNonFaultingZeroExtendToUInt16: + case NI_Sve_LoadVectorByteNonFaultingZeroExtendToUInt32: + case NI_Sve_LoadVectorByteNonFaultingZeroExtendToUInt64: + case NI_Sve_LoadVectorInt16NonFaultingSignExtendToInt32: + case NI_Sve_LoadVectorInt16NonFaultingSignExtendToInt64: + case NI_Sve_LoadVectorInt16NonFaultingSignExtendToUInt32: + case NI_Sve_LoadVectorInt16NonFaultingSignExtendToUInt64: + case NI_Sve_LoadVectorInt32NonFaultingSignExtendToInt64: + case NI_Sve_LoadVectorInt32NonFaultingSignExtendToUInt64: + case NI_Sve_LoadVectorNonFaulting: + case NI_Sve_LoadVectorSByteNonFaultingSignExtendToInt16: + case NI_Sve_LoadVectorSByteNonFaultingSignExtendToInt32: + case NI_Sve_LoadVectorSByteNonFaultingSignExtendToInt64: + case NI_Sve_LoadVectorSByteNonFaultingSignExtendToUInt16: + case NI_Sve_LoadVectorSByteNonFaultingSignExtendToUInt32: + case NI_Sve_LoadVectorSByteNonFaultingSignExtendToUInt64: + case NI_Sve_LoadVectorUInt16NonFaultingZeroExtendToInt32: + case NI_Sve_LoadVectorUInt16NonFaultingZeroExtendToInt64: + case NI_Sve_LoadVectorUInt16NonFaultingZeroExtendToUInt32: + case NI_Sve_LoadVectorUInt16NonFaultingZeroExtendToUInt64: + case NI_Sve_LoadVectorUInt32NonFaultingZeroExtendToInt64: + case NI_Sve_LoadVectorUInt32NonFaultingZeroExtendToUInt64: + { + if (intrin.numOperands == 2) + { + // We have extra argument which means there is a "use" of FFR here. Restore it back in FFR + // register. + assert(op2Reg != REG_NA); + GetEmitter()->emitIns_R(INS_sve_wrffr, emitSize, op2Reg, opt); + } + GetEmitter()->emitIns_R_R(ins, emitSize, targetReg, op1Reg); + break; + } + case NI_Sve_GatherVectorByteZeroExtendFirstFaulting: case NI_Sve_GatherVectorFirstFaulting: case NI_Sve_GatherVectorInt16SignExtendFirstFaulting: diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index 0761c34931ca0..ed238ba0fca56 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -150,12 +150,12 @@ HARDWARE_INTRINSIC(Sve, Load3xVectorAndUnzip, HARDWARE_INTRINSIC(Sve, Load4xVectorAndUnzip, -1, 2, {INS_sve_ld4b, INS_sve_ld4b, INS_sve_ld4h, INS_sve_ld4h, INS_sve_ld4w, INS_sve_ld4w, INS_sve_ld4d, INS_sve_ld4d, INS_sve_ld4w, INS_sve_ld4d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Sve, LoadVector, -1, 2, {INS_sve_ld1b, INS_sve_ld1b, INS_sve_ld1h, INS_sve_ld1h, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1d, INS_sve_ld1d, INS_sve_ld1w, INS_sve_ld1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVector128AndReplicateToVector, -1, 2, {INS_sve_ld1rqb, INS_sve_ld1rqb, INS_sve_ld1rqh, INS_sve_ld1rqh, INS_sve_ld1rqw, INS_sve_ld1rqw, INS_sve_ld1rqd, INS_sve_ld1rqd, INS_sve_ld1rqw, INS_sve_ld1rqd}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToInt16, -1, 1, {INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToInt32, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToInt64, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt16, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt32, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt64, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToInt16, -1, -1, {INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToInt32, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToInt64, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt16, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt32, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt64, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendFirstFaulting, -1, -1, {INS_invalid, INS_invalid, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt16, -1, 2, {INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt32, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) @@ -164,46 +164,46 @@ HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt16, HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt32, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt64, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorFirstFaulting, -1, -1, {INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToInt32, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToInt64, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToUInt32, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToUInt64, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToInt32, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToInt64, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToUInt32, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToUInt64, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendFirstFaulting, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToInt32, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToInt64, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToUInt32, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToUInt64, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorInt32NonFaultingSignExtendToInt64, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sw, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorInt32NonFaultingSignExtendToUInt64, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sw, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorInt32NonFaultingSignExtendToInt64, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sw, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorInt32NonFaultingSignExtendToUInt64, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sw, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, LoadVectorInt32SignExtendFirstFaulting, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sw, INS_sve_ldff1sw, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, LoadVectorInt32SignExtendToInt64, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt32SignExtendToUInt64, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorNonFaulting, -1, 1, {INS_sve_ldnf1b, INS_sve_ldnf1b, INS_sve_ldnf1h, INS_sve_ldnf1h, INS_sve_ldnf1w, INS_sve_ldnf1w, INS_sve_ldnf1d, INS_sve_ldnf1d, INS_sve_ldnf1w, INS_sve_ldnf1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorNonFaulting, -1, -1, {INS_sve_ldnf1b, INS_sve_ldnf1b, INS_sve_ldnf1h, INS_sve_ldnf1h, INS_sve_ldnf1w, INS_sve_ldnf1w, INS_sve_ldnf1d, INS_sve_ldnf1d, INS_sve_ldnf1w, INS_sve_ldnf1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, LoadVectorNonTemporal, -1, 2, {INS_sve_ldnt1b, INS_sve_ldnt1b, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_sve_ldnt1w, INS_sve_ldnt1w, INS_sve_ldnt1d, INS_sve_ldnt1d, INS_sve_ldnt1w, INS_sve_ldnt1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToInt16, -1, -1, {INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToInt32, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToInt64, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt16, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt32, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt64, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendFirstFaulting, -1, -1, {INS_invalid, INS_invalid, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToInt16, -1, -1, {INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToInt32, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToInt64, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt16, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt32, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt64, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendFirstFaulting, -1, -1, {INS_invalid, INS_invalid, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToInt16, -1, 2, {INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToInt32, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToInt64, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToUInt16, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToUInt32, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToUInt64, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToInt32, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToInt64, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToUInt32, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToUInt64, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToInt32, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToInt64, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToUInt32, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToUInt64, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendFirstFaulting, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToInt32, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToInt64, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt32, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt64, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt32NonFaultingZeroExtendToInt64, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1w, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt32NonFaultingZeroExtendToUInt64, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt32NonFaultingZeroExtendToInt64, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1w, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt32NonFaultingZeroExtendToUInt64, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendFirstFaulting, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToInt64, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToUInt64, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index 4090612b9b72a..5b944ff9d5f5a 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -1801,6 +1801,31 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) case NI_Sve_LoadVectorSByteSignExtendFirstFaulting: case NI_Sve_LoadVectorUInt16ZeroExtendFirstFaulting: case NI_Sve_LoadVectorUInt32ZeroExtendFirstFaulting: + case NI_Sve_LoadVectorByteNonFaultingZeroExtendToInt16: + case NI_Sve_LoadVectorByteNonFaultingZeroExtendToInt32: + case NI_Sve_LoadVectorByteNonFaultingZeroExtendToInt64: + case NI_Sve_LoadVectorByteNonFaultingZeroExtendToUInt16: + case NI_Sve_LoadVectorByteNonFaultingZeroExtendToUInt32: + case NI_Sve_LoadVectorByteNonFaultingZeroExtendToUInt64: + case NI_Sve_LoadVectorInt16NonFaultingSignExtendToInt32: + case NI_Sve_LoadVectorInt16NonFaultingSignExtendToInt64: + case NI_Sve_LoadVectorInt16NonFaultingSignExtendToUInt32: + case NI_Sve_LoadVectorInt16NonFaultingSignExtendToUInt64: + case NI_Sve_LoadVectorInt32NonFaultingSignExtendToInt64: + case NI_Sve_LoadVectorInt32NonFaultingSignExtendToUInt64: + case NI_Sve_LoadVectorNonFaulting: + case NI_Sve_LoadVectorSByteNonFaultingSignExtendToInt16: + case NI_Sve_LoadVectorSByteNonFaultingSignExtendToInt32: + case NI_Sve_LoadVectorSByteNonFaultingSignExtendToInt64: + case NI_Sve_LoadVectorSByteNonFaultingSignExtendToUInt16: + case NI_Sve_LoadVectorSByteNonFaultingSignExtendToUInt32: + case NI_Sve_LoadVectorSByteNonFaultingSignExtendToUInt64: + case NI_Sve_LoadVectorUInt16NonFaultingZeroExtendToInt32: + case NI_Sve_LoadVectorUInt16NonFaultingZeroExtendToInt64: + case NI_Sve_LoadVectorUInt16NonFaultingZeroExtendToUInt32: + case NI_Sve_LoadVectorUInt16NonFaultingZeroExtendToUInt64: + case NI_Sve_LoadVectorUInt32NonFaultingZeroExtendToInt64: + case NI_Sve_LoadVectorUInt32NonFaultingZeroExtendToUInt64: { LIR::Use use; bool foundUse = BlockRange().TryGetUse(node, &use); @@ -1821,11 +1846,15 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) assert(node->GetAuxiliaryType() != TYP_UNKNOWN); node->ResetHWIntrinsicId(intrinsicId, comp, node->Op(1), node->Op(2), node->Op(3), lclVar); } - else + else if (node->GetOperandCount() == 2) { - assert(node->GetOperandCount() == 2); node->ResetHWIntrinsicId(intrinsicId, comp, node->Op(1), node->Op(2), lclVar); } + else + { + assert(node->GetOperandCount() == 1); + node->ResetHWIntrinsicId(intrinsicId, comp, node->Op(1), lclVar); + } } if (foundUse) @@ -4151,6 +4180,31 @@ void Lowering::StoreFFRValue(GenTreeHWIntrinsic* node) case NI_Sve_LoadVectorSByteSignExtendFirstFaulting: case NI_Sve_LoadVectorUInt16ZeroExtendFirstFaulting: case NI_Sve_LoadVectorUInt32ZeroExtendFirstFaulting: + case NI_Sve_LoadVectorByteNonFaultingZeroExtendToInt16: + case NI_Sve_LoadVectorByteNonFaultingZeroExtendToInt32: + case NI_Sve_LoadVectorByteNonFaultingZeroExtendToInt64: + case NI_Sve_LoadVectorByteNonFaultingZeroExtendToUInt16: + case NI_Sve_LoadVectorByteNonFaultingZeroExtendToUInt32: + case NI_Sve_LoadVectorByteNonFaultingZeroExtendToUInt64: + case NI_Sve_LoadVectorInt16NonFaultingSignExtendToInt32: + case NI_Sve_LoadVectorInt16NonFaultingSignExtendToInt64: + case NI_Sve_LoadVectorInt16NonFaultingSignExtendToUInt32: + case NI_Sve_LoadVectorInt16NonFaultingSignExtendToUInt64: + case NI_Sve_LoadVectorInt32NonFaultingSignExtendToInt64: + case NI_Sve_LoadVectorInt32NonFaultingSignExtendToUInt64: + case NI_Sve_LoadVectorNonFaulting: + case NI_Sve_LoadVectorSByteNonFaultingSignExtendToInt16: + case NI_Sve_LoadVectorSByteNonFaultingSignExtendToInt32: + case NI_Sve_LoadVectorSByteNonFaultingSignExtendToInt64: + case NI_Sve_LoadVectorSByteNonFaultingSignExtendToUInt16: + case NI_Sve_LoadVectorSByteNonFaultingSignExtendToUInt32: + case NI_Sve_LoadVectorSByteNonFaultingSignExtendToUInt64: + case NI_Sve_LoadVectorUInt16NonFaultingZeroExtendToInt32: + case NI_Sve_LoadVectorUInt16NonFaultingZeroExtendToInt64: + case NI_Sve_LoadVectorUInt16NonFaultingZeroExtendToUInt32: + case NI_Sve_LoadVectorUInt16NonFaultingZeroExtendToUInt64: + case NI_Sve_LoadVectorUInt32NonFaultingZeroExtendToInt64: + case NI_Sve_LoadVectorUInt32NonFaultingZeroExtendToUInt64: case NI_Sve_SetFfr: break; diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index ee6d423db46d9..3c390ee213941 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -3679,13 +3679,18 @@ int LinearScan::BuildOperandUses(GenTree* node, SingleTypeRegSet candidates) { GenTreeHWIntrinsic* hwintrinsic = node->AsHWIntrinsic(); + size_t numArgs = hwintrinsic->GetOperandCount(); if (hwintrinsic->OperIsMemoryLoad()) { +#ifdef TARGET_ARM64 + if (numArgs == 2) + { + return BuildAddrUses(hwintrinsic->Op(1)) + BuildOperandUses(hwintrinsic->Op(2), candidates); + } +#endif return BuildAddrUses(hwintrinsic->Op(1)); } - size_t numArgs = hwintrinsic->GetOperandCount(); - if (numArgs != 1) { #ifdef TARGET_ARM64 diff --git a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs index 05b0757e41a78..457808946c3dd 100644 --- a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs +++ b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs @@ -3809,16 +3809,16 @@ ("SveLoadVectorMaskedTest.template", new Dictionary { ["TestName"] = "Sve_LoadVector_uint", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()"}), ("SveLoadVectorMaskedTest.template", new Dictionary { ["TestName"] = "Sve_LoadVector_ulong", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()"}), - ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_float", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["Cast"] = "(float*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_double", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["Cast"] = "(double*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_sbyte", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["Cast"] = "(sbyte*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_short", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["Cast"] = "(short*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_int", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["Cast"] = "(int*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_long", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["Cast"] = "(long*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_byte", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["Cast"] = "(byte*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_ushort", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["Cast"] = "(ushort*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_uint", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["Cast"] = "(uint*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_ulong", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["Cast"] = "(ulong*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_float", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "64", ["GetFfrType"] = "UInt32", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["Cast"] = "(float*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_double", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["GetFfrType"] = "UInt64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["Cast"] = "(double*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_sbyte", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["GetFfrType"] = "SByte", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["Cast"] = "(sbyte*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_short", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["GetFfrType"] = "Int16", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["Cast"] = "(short*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_int", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["GetFfrType"] = "Int32", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["Cast"] = "(int*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_long", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["GetFfrType"] = "Int64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["Cast"] = "(long*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_byte", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["GetFfrType"] = "Byte", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["Cast"] = "(byte*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_ushort", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["GetFfrType"] = "UInt16", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["Cast"] = "(ushort*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_uint", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["GetFfrType"] = "UInt32", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["Cast"] = "(uint*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_ulong", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["GetFfrType"] = "UInt64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["Cast"] = "(ulong*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorByteZeroExtendFirstFaulting_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorByteZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Byte", ["GetFfrType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()"}), ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorByteZeroExtendFirstFaulting_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorByteZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Byte", ["GetFfrType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()"}), @@ -3883,18 +3883,18 @@ ("SveLoadVectorMaskedTest.template", new Dictionary { ["TestName"] = "Sve_LoadVector128AndReplicateToVector_uint", ["Isa"] = "Sve", ["Method"] = "LoadVector128AndReplicateToVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()"}), ("SveLoadVectorMaskedTest.template", new Dictionary { ["TestName"] = "Sve_LoadVector128AndReplicateToVector_ulong", ["Isa"] = "Sve", ["Method"] = "LoadVector128AndReplicateToVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()"}), - ("SveLoadNonFaultingUnOpTest.template",new Dictionary {["TestName"] = "Sve_LoadVectorInt16NonFaultingSignExtendToInt32_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorInt16NonFaultingSignExtendToInt32", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]",}), - ("SveLoadNonFaultingUnOpTest.template",new Dictionary {["TestName"] = "Sve_LoadVectorInt16NonFaultingSignExtendToInt64_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorInt16NonFaultingSignExtendToInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]",}), - ("SveLoadNonFaultingUnOpTest.template",new Dictionary {["TestName"] = "Sve_LoadVectorInt16NonFaultingSignExtendToUInt32_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorInt16NonFaultingSignExtendToUInt32", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["Cast"] = "", ["ValidateIterResult"] = "((uint)firstOp[i]) != result[i]",}), - ("SveLoadNonFaultingUnOpTest.template",new Dictionary {["TestName"] = "Sve_LoadVectorInt16NonFaultingSignExtendToUInt64_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorInt16NonFaultingSignExtendToUInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["Cast"] = "", ["ValidateIterResult"] = "((ulong)firstOp[i]) != result[i]",}), - ("SveLoadNonFaultingUnOpTest.template",new Dictionary {["TestName"] = "Sve_LoadVectorInt32NonFaultingSignExtendToInt64_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorInt32NonFaultingSignExtendToInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]",}), - ("SveLoadNonFaultingUnOpTest.template",new Dictionary {["TestName"] = "Sve_LoadVectorInt32NonFaultingSignExtendToUInt64_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorInt32NonFaultingSignExtendToUInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["Cast"] = "", ["ValidateIterResult"] = "((ulong)firstOp[i]) != result[i]",}), - ("SveLoadNonFaultingUnOpTest.template",new Dictionary {["TestName"] = "Sve_LoadVectorSByteNonFaultingSignExtendToInt16_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorSByteNonFaultingSignExtendToInt16", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]",}), - ("SveLoadNonFaultingUnOpTest.template",new Dictionary {["TestName"] = "Sve_LoadVectorSByteNonFaultingSignExtendToInt32_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorSByteNonFaultingSignExtendToInt32", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]",}), - ("SveLoadNonFaultingUnOpTest.template",new Dictionary {["TestName"] = "Sve_LoadVectorSByteNonFaultingSignExtendToInt64_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorSByteNonFaultingSignExtendToInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]",}), - ("SveLoadNonFaultingUnOpTest.template",new Dictionary {["TestName"] = "Sve_LoadVectorSByteNonFaultingSignExtendToUInt16_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorSByteNonFaultingSignExtendToUInt16", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["Cast"] = "", ["ValidateIterResult"] = "((ushort)firstOp[i]) != result[i]",}), - ("SveLoadNonFaultingUnOpTest.template",new Dictionary {["TestName"] = "Sve_LoadVectorSByteNonFaultingSignExtendToUInt32_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorSByteNonFaultingSignExtendToUInt32", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["Cast"] = "", ["ValidateIterResult"] = "((uint)firstOp[i]) != result[i]",}), - ("SveLoadNonFaultingUnOpTest.template",new Dictionary {["TestName"] = "Sve_LoadVectorSByteNonFaultingSignExtendToUInt64_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorSByteNonFaultingSignExtendToUInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["Cast"] = "", ["ValidateIterResult"] = "((ulong)firstOp[i]) != result[i]",}), + ("SveLoadNonFaultingUnOpTest.template",new Dictionary {["TestName"] = "Sve_LoadVectorInt16NonFaultingSignExtendToInt32_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorInt16NonFaultingSignExtendToInt32", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["GetFfrType"] = "Int32", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]",}), + ("SveLoadNonFaultingUnOpTest.template",new Dictionary {["TestName"] = "Sve_LoadVectorInt16NonFaultingSignExtendToInt64_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorInt16NonFaultingSignExtendToInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["GetFfrType"] = "Int64", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]",}), + ("SveLoadNonFaultingUnOpTest.template",new Dictionary {["TestName"] = "Sve_LoadVectorInt16NonFaultingSignExtendToUInt32_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorInt16NonFaultingSignExtendToUInt32", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["GetFfrType"] = "UInt32", ["Cast"] = "", ["ValidateIterResult"] = "((uint)firstOp[i]) != result[i]",}), + ("SveLoadNonFaultingUnOpTest.template",new Dictionary {["TestName"] = "Sve_LoadVectorInt16NonFaultingSignExtendToUInt64_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorInt16NonFaultingSignExtendToUInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["GetFfrType"] = "UInt64", ["Cast"] = "", ["ValidateIterResult"] = "((ulong)firstOp[i]) != result[i]",}), + ("SveLoadNonFaultingUnOpTest.template",new Dictionary {["TestName"] = "Sve_LoadVectorInt32NonFaultingSignExtendToInt64_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorInt32NonFaultingSignExtendToInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["GetFfrType"] = "Int64", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]",}), + ("SveLoadNonFaultingUnOpTest.template",new Dictionary {["TestName"] = "Sve_LoadVectorInt32NonFaultingSignExtendToUInt64_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorInt32NonFaultingSignExtendToUInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["GetFfrType"] = "UInt64", ["Cast"] = "", ["ValidateIterResult"] = "((ulong)firstOp[i]) != result[i]",}), + ("SveLoadNonFaultingUnOpTest.template",new Dictionary {["TestName"] = "Sve_LoadVectorSByteNonFaultingSignExtendToInt16_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorSByteNonFaultingSignExtendToInt16", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["GetFfrType"] = "Int16", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]",}), + ("SveLoadNonFaultingUnOpTest.template",new Dictionary {["TestName"] = "Sve_LoadVectorSByteNonFaultingSignExtendToInt32_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorSByteNonFaultingSignExtendToInt32", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["GetFfrType"] = "Int32", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]",}), + ("SveLoadNonFaultingUnOpTest.template",new Dictionary {["TestName"] = "Sve_LoadVectorSByteNonFaultingSignExtendToInt64_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorSByteNonFaultingSignExtendToInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["GetFfrType"] = "Int64", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]",}), + ("SveLoadNonFaultingUnOpTest.template",new Dictionary {["TestName"] = "Sve_LoadVectorSByteNonFaultingSignExtendToUInt16_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorSByteNonFaultingSignExtendToUInt16", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["GetFfrType"] = "UInt16", ["Cast"] = "", ["ValidateIterResult"] = "((ushort)firstOp[i]) != result[i]",}), + ("SveLoadNonFaultingUnOpTest.template",new Dictionary {["TestName"] = "Sve_LoadVectorSByteNonFaultingSignExtendToUInt32_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorSByteNonFaultingSignExtendToUInt32", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["GetFfrType"] = "UInt32", ["Cast"] = "", ["ValidateIterResult"] = "((uint)firstOp[i]) != result[i]",}), + ("SveLoadNonFaultingUnOpTest.template",new Dictionary {["TestName"] = "Sve_LoadVectorSByteNonFaultingSignExtendToUInt64_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorSByteNonFaultingSignExtendToUInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["GetFfrType"] = "UInt64", ["Cast"] = "", ["ValidateIterResult"] = "((ulong)firstOp[i]) != result[i]",}), ("SveLoadVectorMaskedTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorInt16SignExtendToInt32", ["Isa"] = "Sve", ["Method"] = "LoadVectorInt16SignExtendToInt32", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()"}), ("SveLoadVectorMaskedTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorInt16SignExtendToInt64", ["Isa"] = "Sve", ["Method"] = "LoadVectorInt16SignExtendToInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()"}), @@ -3921,18 +3921,18 @@ ("SveLoadVectorMaskedTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorUInt32ZeroExtendToInt64", ["Isa"] = "Sve", ["Method"] = "LoadVectorUInt32ZeroExtendToInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()"}), ("SveLoadVectorMaskedTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorUInt32ZeroExtendToUInt64", ["Isa"] = "Sve", ["Method"] = "LoadVectorUInt32ZeroExtendToUInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()"}), - ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorByteNonFaultingZeroExtendToInt16", ["Isa"] = "Sve", ["Method"] = "LoadVectorByteNonFaultingZeroExtendToInt16", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorByteNonFaultingZeroExtendToInt32", ["Isa"] = "Sve", ["Method"] = "LoadVectorByteNonFaultingZeroExtendToInt32", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorByteNonFaultingZeroExtendToInt64", ["Isa"] = "Sve", ["Method"] = "LoadVectorByteNonFaultingZeroExtendToInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorByteNonFaultingZeroExtendToUInt16", ["Isa"] = "Sve", ["Method"] = "LoadVectorByteNonFaultingZeroExtendToUInt16", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorByteNonFaultingZeroExtendToUInt32", ["Isa"] = "Sve", ["Method"] = "LoadVectorByteNonFaultingZeroExtendToUInt32", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorByteNonFaultingZeroExtendToUInt64", ["Isa"] = "Sve", ["Method"] = "LoadVectorByteNonFaultingZeroExtendToUInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorUInt16NonFaultingZeroExtendToInt32", ["Isa"] = "Sve", ["Method"] = "LoadVectorUInt16NonFaultingZeroExtendToInt32", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorUInt16NonFaultingZeroExtendToInt64", ["Isa"] = "Sve", ["Method"] = "LoadVectorUInt16NonFaultingZeroExtendToInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorUInt16NonFaultingZeroExtendToUInt32", ["Isa"] = "Sve", ["Method"] = "LoadVectorUInt16NonFaultingZeroExtendToUInt32", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorUInt16NonFaultingZeroExtendToUInt64", ["Isa"] = "Sve", ["Method"] = "LoadVectorUInt16NonFaultingZeroExtendToUInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorUInt32NonFaultingZeroExtendToInt64", ["Isa"] = "Sve", ["Method"] = "LoadVectorUInt32NonFaultingZeroExtendToInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorUInt32NonFaultingZeroExtendToUInt64", ["Isa"] = "Sve", ["Method"] = "LoadVectorUInt32NonFaultingZeroExtendToUInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorByteNonFaultingZeroExtendToInt16", ["Isa"] = "Sve", ["Method"] = "LoadVectorByteNonFaultingZeroExtendToInt16", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["GetFfrType"] = "Int16", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorByteNonFaultingZeroExtendToInt32", ["Isa"] = "Sve", ["Method"] = "LoadVectorByteNonFaultingZeroExtendToInt32", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["GetFfrType"] = "Int32", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorByteNonFaultingZeroExtendToInt64", ["Isa"] = "Sve", ["Method"] = "LoadVectorByteNonFaultingZeroExtendToInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["GetFfrType"] = "Int64", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorByteNonFaultingZeroExtendToUInt16", ["Isa"] = "Sve", ["Method"] = "LoadVectorByteNonFaultingZeroExtendToUInt16", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["GetFfrType"] = "UInt16", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorByteNonFaultingZeroExtendToUInt32", ["Isa"] = "Sve", ["Method"] = "LoadVectorByteNonFaultingZeroExtendToUInt32", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["GetFfrType"] = "UInt32", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorByteNonFaultingZeroExtendToUInt64", ["Isa"] = "Sve", ["Method"] = "LoadVectorByteNonFaultingZeroExtendToUInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["GetFfrType"] = "UInt64", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorUInt16NonFaultingZeroExtendToInt32", ["Isa"] = "Sve", ["Method"] = "LoadVectorUInt16NonFaultingZeroExtendToInt32", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["GetFfrType"] = "Int32", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorUInt16NonFaultingZeroExtendToInt64", ["Isa"] = "Sve", ["Method"] = "LoadVectorUInt16NonFaultingZeroExtendToInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["GetFfrType"] = "Int64", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorUInt16NonFaultingZeroExtendToUInt32", ["Isa"] = "Sve", ["Method"] = "LoadVectorUInt16NonFaultingZeroExtendToUInt32", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["GetFfrType"] = "UInt32", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorUInt16NonFaultingZeroExtendToUInt64", ["Isa"] = "Sve", ["Method"] = "LoadVectorUInt16NonFaultingZeroExtendToUInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["GetFfrType"] = "UInt64", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorUInt32NonFaultingZeroExtendToInt64", ["Isa"] = "Sve", ["Method"] = "LoadVectorUInt32NonFaultingZeroExtendToInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["GetFfrType"] = "Int64", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorUInt32NonFaultingZeroExtendToUInt64", ["Isa"] = "Sve", ["Method"] = "LoadVectorUInt32NonFaultingZeroExtendToUInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["GetFfrType"] = "UInt64", ["Cast"] = "", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), ("SveLoad2xVectorAndUnzipTest.template", new Dictionary { ["TestName"] = "Sve_Load2xVectorAndUnzip_float", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Load2xVectorAndUnzip", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateIterResult"] = "result1[i] != input[i * 2] || result2[i] != input[i * 2 + 1]"}), ("SveLoad2xVectorAndUnzipTest.template", new Dictionary { ["TestName"] = "Sve_Load2xVectorAndUnzip_double", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Load2xVectorAndUnzip", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateIterResult"] = "result1[i] != input[i * 2] || result2[i] != input[i * 2 + 1]"}), diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadNonFaultingUnOpTest.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadNonFaultingUnOpTest.template index 99cda77edff66..f5364238d58e0 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadNonFaultingUnOpTest.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadNonFaultingUnOpTest.template @@ -9,6 +9,7 @@ ******************************************************************************/ using System; +using System.Buffers; using System.Numerics; using System.Reflection; using System.Runtime.CompilerServices; @@ -31,7 +32,7 @@ namespace JIT.HardwareIntrinsics.Arm // Validates basic functionality works test.RunBasicScenario_Load(); - // Validates basic functionality + // Validates basic functionality test.RunBasicScenario_LoadNonFaulting(); // Validates calling via reflection works @@ -81,6 +82,9 @@ namespace JIT.HardwareIntrinsics.Arm { int sizeOfinArray1 = inArray1.Length * Unsafe.SizeOf<{Op1BaseType}>(); int sizeOfoutArray = outArray.Length * Unsafe.SizeOf<{RetBaseType}>(); + // Choose size of bounded memory such that half of the vector would fall in accessible memory while the rest would in non-accessible one. + int sizeOfInBounded = {RetVectorType}<{RetBaseType}>.Count / 2; + if ((alignment != 64 && alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfinArray1 || (alignment * 2) < sizeOfoutArray) { throw new ArgumentException($"Invalid value of alignment: {alignment}, sizeOfinArray1: {sizeOfinArray1}, sizeOfoutArray: {sizeOfoutArray}"); @@ -93,11 +97,14 @@ namespace JIT.HardwareIntrinsics.Arm this.outHandle = GCHandle.Alloc(this.outArray, GCHandleType.Pinned); this.alignment = (ulong)alignment; + this.inBounded = BoundedMemory.Allocate(sizeOfInBounded, PoisonPagePlacement.After); Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray1Ptr), ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), (uint)sizeOfinArray1); - + Unsafe.CopyBlockUnaligned(ref inBounded.Span.GetPinnableReference(), ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), (uint)sizeOfInBounded); } + public BoundedMemory inBounded; + public void* inArray1Ptr => Align((byte*)(inHandle1.AddrOfPinnedObject().ToPointer()), alignment); public void* outArrayPtr => Align((byte*)(outHandle.AddrOfPinnedObject().ToPointer()), alignment); @@ -134,7 +141,7 @@ namespace JIT.HardwareIntrinsics.Arm Unsafe.Write(testClass._dataTable.outArrayPtr, result); testClass.ValidateResult(testClass._dataTable.inArray1Ptr, testClass._dataTable.outArrayPtr); } - } + } private static readonly int LargestVectorSize = {LargestVectorSize}; @@ -185,10 +192,14 @@ namespace JIT.HardwareIntrinsics.Arm { TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_LoadNonFaulting)); - var result = {Isa}.{Method}({Cast}default); + ref var op1Ref = ref _dataTable.inBounded.Span.GetPinnableReference(); + + Sve.SetFfr(Sve.CreateTrueMaskByte(SveMaskPattern.All)); + var result = {Isa}.{Method}(({Op1BaseType}*)(Unsafe.AsPointer(ref op1Ref))); + var faultResult = Sve.GetFfr{GetFfrType}(); Unsafe.Write(_dataTable.outArrayPtr, result); - ValidateZeroResult(_dataTable.inArray1Ptr, _dataTable.outArrayPtr); + ValidateNonFaultingResult(Unsafe.AsPointer(ref op1Ref), _dataTable.outArrayPtr, faultResult); } public void RunReflectionScenario_Load() @@ -237,19 +248,19 @@ namespace JIT.HardwareIntrinsics.Arm { TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_mask - operation in TrueValue"); ConditionalSelectScenario_TrueValue(_mask, _dataTable.inArray1Ptr, _falseFld); - + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_zero - operation in TrueValue"); ConditionalSelectScenario_TrueValue({Op1VectorType}<{RetBaseType}>.Zero, _dataTable.inArray1Ptr, _falseFld); - + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_all - operation in TrueValue"); ConditionalSelectScenario_TrueValue({Op1VectorType}<{RetBaseType}>.AllBitsSet, _dataTable.inArray1Ptr, _falseFld); TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_mask - operation in FalseValue"); ConditionalSelectScenario_FalseValue(_mask, _dataTable.inArray1Ptr, _falseFld); - + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_zero - operation in FalseValue"); ConditionalSelectScenario_FalseValue({Op1VectorType}<{RetBaseType}>.Zero, _dataTable.inArray1Ptr, _falseFld); - + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_all - operation in FalseValue"); ConditionalSelectScenario_FalseValue({Op1VectorType}<{RetBaseType}>.AllBitsSet, _dataTable.inArray1Ptr, _falseFld); } @@ -258,19 +269,19 @@ namespace JIT.HardwareIntrinsics.Arm { TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_mask - operation in TrueValue"); ConditionalSelectScenario_TrueValue(_mask, _dataTable.inArray1Ptr, {Op1VectorType}<{RetBaseType}>.Zero); - + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero - operation in TrueValue"); ConditionalSelectScenario_TrueValue({Op1VectorType}<{RetBaseType}>.Zero, _dataTable.inArray1Ptr, {Op1VectorType}<{RetBaseType}>.Zero); - + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all - operation in TrueValue"); ConditionalSelectScenario_TrueValue({Op1VectorType}<{RetBaseType}>.AllBitsSet, _dataTable.inArray1Ptr, {Op1VectorType}<{RetBaseType}>.Zero); TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_mask - operation in FalseValue"); ConditionalSelectScenario_FalseValue(_mask, _dataTable.inArray1Ptr, {Op1VectorType}<{RetBaseType}>.Zero); - + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero - operation in FalseValue"); ConditionalSelectScenario_FalseValue({Op1VectorType}<{RetBaseType}>.Zero, _dataTable.inArray1Ptr, {Op1VectorType}<{RetBaseType}>.Zero); - + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all - operation in FalseValue"); ConditionalSelectScenario_FalseValue({Op1VectorType}<{RetBaseType}>.AllBitsSet, _dataTable.inArray1Ptr, {Op1VectorType}<{RetBaseType}>.Zero); } @@ -292,7 +303,7 @@ namespace JIT.HardwareIntrinsics.Arm Unsafe.Write(_dataTable.outArrayPtr, result); ValidateConditionalSelectResult_FalseValue(mask, address, trueOp, _dataTable.outArrayPtr); } - + public void RunUnsupportedScenario() { TestLibrary.TestFramework.BeginScenario(nameof(RunUnsupportedScenario)); @@ -443,6 +454,46 @@ namespace JIT.HardwareIntrinsics.Arm ValidateZeroResult(inArray, outArray, method); } + private void ValidateNonFaultingResult(void* firstOp, void* result, Vector<{GetFfrType}> faultResult, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray = new {Op1BaseType}[Op1ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + {GetFfrType}[] faultArray = new {GetFfrType}[RetElementCount]; + + // Only read bytes mapping to half of the destination vector. The remaining elements are in the poison page. + // It would make inArray to contain elements that could read without a fault followed by zeros. + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray[0]), ref Unsafe.AsRef(firstOp), ((uint){RetVectorType}<{RetBaseType}>.Count / 2) ); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + fixed({GetFfrType}* faultArrayPtr = &faultArray[0]) + { + Unsafe.Write(faultArrayPtr, faultResult); + } + + bool succeeded = true; + + for (var i = 0; i < RetElementCount; i++) + { + var expectedOut = ({RetBaseType})((i < (RetElementCount / 2)) ? inArray[i] : 0); + var expectedFault = ({GetFfrType}) ((i < (RetElementCount / 2)) ? 1 : 0); + + if ((outArray[i] != expectedOut) && (faultArray[i] != expectedFault) ) + { + succeeded = false; + break; + } + } + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1BaseType}): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", inArray)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", outArray)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + Succeeded = false; + } + } + private void ValidateZeroResult({Op1BaseType}[] firstOp, {RetBaseType}[] result, [CallerMemberName] string method = "") { bool succeeded = true;