From 4589e7c4ed79d1efd93a850aa943b0c433645d57 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Sun, 31 Oct 2021 14:16:54 +0300 Subject: [PATCH 01/12] arm64: recognize sbfiz/ubfiz idioms --- src/coreclr/jit/codegenarmarch.cpp | 14 +++++++++++++- src/coreclr/jit/codegenlinear.cpp | 2 +- src/coreclr/jit/lower.cpp | 19 +++++++++++++++++++ src/coreclr/jit/lsraarm64.cpp | 11 ++++++++++- 4 files changed, 43 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 01b28767db5dd..62b2ed5d70619 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -1630,7 +1630,19 @@ void CodeGen::genCodeForShift(GenTree* tree) unsigned immWidth = emitter::getBitWidth(size); // For ARM64, immWidth will be set to 32 or 64 unsigned shiftByImm = (unsigned)shiftBy->AsIntCon()->gtIconVal & (immWidth - 1); - GetEmitter()->emitIns_R_R_I(ins, size, tree->GetRegNum(), operand->GetRegNum(), shiftByImm); + // Check if it's a sbfiz/ubfiz idiom (e.g. '(ulong)x << 2') + if (tree->gtGetOp1()->OperIs(GT_CAST) && tree->gtGetOp1()->isContained()) + { + GenTreeCast* cast = tree->gtGetOp1()->AsCast(); + assert((shiftByImm > 0) && (shiftByImm < 32)); + assert(varTypeToSigned(cast->CastFromType()) == TYP_INT); + GetEmitter()->emitIns_R_R_I_I(cast->IsUnsigned() ? INS_ubfiz : INS_sbfiz, size, tree->GetRegNum(), + cast->CastOp()->GetRegNum(), (int)shiftByImm, 32); + } + else + { + GetEmitter()->emitIns_R_R_I(ins, size, tree->GetRegNum(), operand->GetRegNum(), shiftByImm); + } } genProduceReg(tree); diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index c9251065ce5b9..5093c74c34a48 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -1608,7 +1608,7 @@ void CodeGen::genConsumeRegs(GenTree* tree) } #endif // FEATURE_HW_INTRINSICS #endif // TARGET_XARCH - else if (tree->OperIs(GT_BITCAST)) + else if (tree->OperIs(GT_BITCAST, GT_CAST)) { genConsumeReg(tree->gtGetOp1()); } diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index 2c9ed01a1ebeb..fe7b410233fd5 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -5753,6 +5753,25 @@ void Lowering::LowerShift(GenTreeOp* shift) shift->gtOp2->ClearContained(); } ContainCheckShiftRotate(shift); + +// Check if we can fold e.g. '(ulong/long)x << 2' where x is int/uint to ubfiz/sbfiz +#ifdef TARGET_ARM64 + if (comp->opts.OptimizationEnabled() && shift->OperIs(GT_LSH) && shift->gtGetOp1()->OperIs(GT_CAST) && + shift->gtGetOp2()->IsCnsIntOrI() && !shift->isContained() && varTypeIsIntegral(shift)) + { + GenTreeIntCon* cns = shift->gtGetOp2()->AsIntCon(); + GenTreeCast* cast = shift->gtGetOp1()->AsCast(); + + if (!cast->isContained() && !cast->IsRegOptional() && varTypeIsLong(shift) && !cast->CastOp()->isContained() && + (varTypeToSigned(cast->CastFromType()) == TYP_INT) && (cns->IconValue() > 0) && (cns->IconValue() < 32)) + { + // 1-31 constant should already be contained at this point + assert(cns->isContained()); + JITDUMP("Recognized ubfix/sbfix pattern in LSH(CAST, CNS), marking CAST node as contained."); + MakeSrcContained(shift, cast); + } + } +#endif } void Lowering::WidenSIMD12IfNecessary(GenTreeLclVarCommon* node) diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 1c66f74d14584..f6cbaf6424dbb 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -275,7 +275,16 @@ int LinearScan::BuildNode(GenTree* tree) case GT_RSH: case GT_RSZ: case GT_ROR: - srcCount = BuildBinaryUses(tree->AsOp()); + if (tree->gtGetOp1()->isContained()) + { + assert(tree->OperIs(GT_LSH) && tree->gtGetOp1()->OperIs(GT_CAST) && tree->gtGetOp2()->IsCnsIntOrI()); + BuildUse(tree->gtGetOp1()->gtGetOp1()); + srcCount = 1; + } + else + { + srcCount = BuildBinaryUses(tree->AsOp()); + } assert(dstCount == 1); BuildDef(tree); break; From d38830488e1c71946fabb2b2efdd01f990ced8c0 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Sun, 31 Oct 2021 14:34:38 +0300 Subject: [PATCH 02/12] fix arm32 build --- src/coreclr/jit/codegenarmarch.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 62b2ed5d70619..1ac3dfe2f9647 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -1630,6 +1630,7 @@ void CodeGen::genCodeForShift(GenTree* tree) unsigned immWidth = emitter::getBitWidth(size); // For ARM64, immWidth will be set to 32 or 64 unsigned shiftByImm = (unsigned)shiftBy->AsIntCon()->gtIconVal & (immWidth - 1); +#ifdef TARGET_ARM64 // Check if it's a sbfiz/ubfiz idiom (e.g. '(ulong)x << 2') if (tree->gtGetOp1()->OperIs(GT_CAST) && tree->gtGetOp1()->isContained()) { @@ -1640,6 +1641,7 @@ void CodeGen::genCodeForShift(GenTree* tree) cast->CastOp()->GetRegNum(), (int)shiftByImm, 32); } else +#endif { GetEmitter()->emitIns_R_R_I(ins, size, tree->GetRegNum(), operand->GetRegNum(), shiftByImm); } From 1763f125b3bd422df81bde865b34368ca909a2e7 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Sun, 31 Oct 2021 14:38:17 +0300 Subject: [PATCH 03/12] Add additional asserts --- src/coreclr/jit/codegenarmarch.cpp | 3 ++- src/coreclr/jit/lower.cpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 1ac3dfe2f9647..df81b5d54d47c 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -1636,7 +1636,8 @@ void CodeGen::genCodeForShift(GenTree* tree) { GenTreeCast* cast = tree->gtGetOp1()->AsCast(); assert((shiftByImm > 0) && (shiftByImm < 32)); - assert(varTypeToSigned(cast->CastFromType()) == TYP_INT); + assert(varTypeIsLong(tree) && (varTypeToSigned(cast->CastFromType()) == TYP_INT)); + assert(!cast->gtOverflow()); GetEmitter()->emitIns_R_R_I_I(cast->IsUnsigned() ? INS_ubfiz : INS_sbfiz, size, tree->GetRegNum(), cast->CastOp()->GetRegNum(), (int)shiftByImm, 32); } diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index fe7b410233fd5..3828302ec29ca 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -5763,7 +5763,8 @@ void Lowering::LowerShift(GenTreeOp* shift) GenTreeCast* cast = shift->gtGetOp1()->AsCast(); if (!cast->isContained() && !cast->IsRegOptional() && varTypeIsLong(shift) && !cast->CastOp()->isContained() && - (varTypeToSigned(cast->CastFromType()) == TYP_INT) && (cns->IconValue() > 0) && (cns->IconValue() < 32)) + (varTypeToSigned(cast->CastFromType()) == TYP_INT) && (cns->IconValue() > 0) && (cns->IconValue() < 32) && + !cast->gtOverflow() && varTypeIsIntegral(cast->CastOp())) { // 1-31 constant should already be contained at this point assert(cns->isContained()); From 42cc66fdafb30ccaf0a0b724cd3551023bb3f7d6 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Sun, 31 Oct 2021 17:56:56 +0300 Subject: [PATCH 04/12] Address feedback & handle small integers --- src/coreclr/jit/codegenarmarch.cpp | 27 +++++++++++++++----- src/coreclr/jit/lower.cpp | 41 ++++++++++++++++++++++++------ 2 files changed, 54 insertions(+), 14 deletions(-) diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index df81b5d54d47c..472ccd35f84e5 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -1631,15 +1631,30 @@ void CodeGen::genCodeForShift(GenTree* tree) unsigned shiftByImm = (unsigned)shiftBy->AsIntCon()->gtIconVal & (immWidth - 1); #ifdef TARGET_ARM64 - // Check if it's a sbfiz/ubfiz idiom (e.g. '(ulong)x << 2') + // Check if we can recognize ubfiz/sbfiz idiom in LSH(CAST(X), CNS) pattern if (tree->gtGetOp1()->OperIs(GT_CAST) && tree->gtGetOp1()->isContained()) { - GenTreeCast* cast = tree->gtGetOp1()->AsCast(); - assert((shiftByImm > 0) && (shiftByImm < 32)); - assert(varTypeIsLong(tree) && (varTypeToSigned(cast->CastFromType()) == TYP_INT)); - assert(!cast->gtOverflow()); + assert(shiftBy->IsCnsIntOrI()); + GenTreeCast* cast = tree->gtGetOp1()->AsCast(); + int width = 0; + if (varTypeIsSmall(cast->CastToType())) + { + // TYP_INT <- %SMALL_INT% <- TYP_INT + // TYP_INT <- %SMALL_UINT% <- TYP_INT + width = (int)genTypeSize(cast->CastToType()) * 8; + assert((width == 16) || (width == 8)); + assert((cast->CastFromType() == TYP_INT) && cast->TypeIs(TYP_INT)); + } + else + { + // TYP_LONG <- TYP_INT + // TYP_LONG <- TYP_UINT <- TYP_INT + // TYP_LONG <- TYP_ULONG <- TYP_INT + width = 32; //(int)genTypeSize(cast->CastFromType()) * 8; + assert((cast->CastFromType() == TYP_INT) && cast->TypeIs(TYP_LONG)); + } GetEmitter()->emitIns_R_R_I_I(cast->IsUnsigned() ? INS_ubfiz : INS_sbfiz, size, tree->GetRegNum(), - cast->CastOp()->GetRegNum(), (int)shiftByImm, 32); + cast->CastOp()->GetRegNum(), (int)shiftByImm, width); } else #endif diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index 3828302ec29ca..bcba8e86f8553 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -5754,22 +5754,47 @@ void Lowering::LowerShift(GenTreeOp* shift) } ContainCheckShiftRotate(shift); -// Check if we can fold e.g. '(ulong/long)x << 2' where x is int/uint to ubfiz/sbfiz #ifdef TARGET_ARM64 + // Check if we can recognize ubfiz/sbfiz idiom in LSH(CAST(X), CNS) pattern if (comp->opts.OptimizationEnabled() && shift->OperIs(GT_LSH) && shift->gtGetOp1()->OperIs(GT_CAST) && shift->gtGetOp2()->IsCnsIntOrI() && !shift->isContained() && varTypeIsIntegral(shift)) { GenTreeIntCon* cns = shift->gtGetOp2()->AsIntCon(); GenTreeCast* cast = shift->gtGetOp1()->AsCast(); - if (!cast->isContained() && !cast->IsRegOptional() && varTypeIsLong(shift) && !cast->CastOp()->isContained() && - (varTypeToSigned(cast->CastFromType()) == TYP_INT) && (cns->IconValue() > 0) && (cns->IconValue() < 32) && - !cast->gtOverflow() && varTypeIsIntegral(cast->CastOp())) + if (!cast->isContained() && !cast->IsRegOptional() && !cast->gtOverflow() && + // CastOp can be a small int in case of CAST(IND(X)) + !varTypeIsSmall(cast->CastOp())) { - // 1-31 constant should already be contained at this point - assert(cns->isContained()); - JITDUMP("Recognized ubfix/sbfix pattern in LSH(CAST, CNS), marking CAST node as contained."); - MakeSrcContained(shift, cast); + // Cast can be either "TYP_LONG <- TYP_INT" or "TYP_INT <- %SMALL_INT% <- TYP_INT" + UINT32 treeSize = genTypeSize(cast); + UINT32 toSize = genTypeSize(cast->CastToType()); + UINT32 fromSize = genTypeSize(cast->CastFromType()); + + assert(varTypeIsIntegral(cast->CastToType())); // Just to make sure there are no surprises here + assert(!cast->CastOp()->isContained()); + + if (treeSize >= fromSize) // It has to be an upcast + { + if (toSize < fromSize) + { + // Special case: "TYP_INT <- %SMALL_INT% <- TYP_INT" + assert(varTypeIsSmall(cast->CastToType()) && (fromSize == 4)); + toSize = 4; + } + assert(toSize >= 4); + + // LSH(TYP_LONG <- TYP_INT, CNS) => 1..31 + // LSH(TYP_INT <- TYP_SHORT <- TYP_INT, CNS) => 1..15 + // LSH(TYP_INT <- TYP_BYTE <- TYP_INT, CNS) => 1..7 + if ((UINT32)cns->IconValue() < (toSize * 4)) + { + // cns is small and should already be contained at this point + assert(cns->isContained()); + JITDUMP("Recognized ubfix/sbfix pattern in LSH(CAST, CNS), marking CAST node as contained."); + MakeSrcContained(shift, cast); + } + } } } #endif From 1ea0e8196c383b89442f6020f7ff4f6f0d72eb56 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Sun, 31 Oct 2021 18:23:04 +0300 Subject: [PATCH 05/12] Clean up --- src/coreclr/jit/codegenarmarch.cpp | 4 +--- src/coreclr/jit/lower.cpp | 13 ++++++------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 472ccd35f84e5..a57c208d48df2 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -1634,13 +1634,11 @@ void CodeGen::genCodeForShift(GenTree* tree) // Check if we can recognize ubfiz/sbfiz idiom in LSH(CAST(X), CNS) pattern if (tree->gtGetOp1()->OperIs(GT_CAST) && tree->gtGetOp1()->isContained()) { - assert(shiftBy->IsCnsIntOrI()); GenTreeCast* cast = tree->gtGetOp1()->AsCast(); int width = 0; if (varTypeIsSmall(cast->CastToType())) { - // TYP_INT <- %SMALL_INT% <- TYP_INT - // TYP_INT <- %SMALL_UINT% <- TYP_INT + // TYP_INT <- %SMALL_(U)INT% <- TYP_INT width = (int)genTypeSize(cast->CastToType()) * 8; assert((width == 16) || (width == 8)); assert((cast->CastFromType() == TYP_INT) && cast->TypeIs(TYP_INT)); diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index bcba8e86f8553..2689470a42a0f 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -5763,30 +5763,29 @@ void Lowering::LowerShift(GenTreeOp* shift) GenTreeCast* cast = shift->gtGetOp1()->AsCast(); if (!cast->isContained() && !cast->IsRegOptional() && !cast->gtOverflow() && - // CastOp can be a small int in case of CAST(IND(X)) - !varTypeIsSmall(cast->CastOp())) + // CastOp should not be a small int, e.g. CAST(IND(ByteTarget)) + cast->CastOp()->TypeIs(TYP_LONG, TYP_INT)) { // Cast can be either "TYP_LONG <- TYP_INT" or "TYP_INT <- %SMALL_INT% <- TYP_INT" UINT32 treeSize = genTypeSize(cast); - UINT32 toSize = genTypeSize(cast->CastToType()); UINT32 fromSize = genTypeSize(cast->CastFromType()); + UINT32 toSize = genTypeSize(cast->CastToType()); - assert(varTypeIsIntegral(cast->CastToType())); // Just to make sure there are no surprises here assert(!cast->CastOp()->isContained()); if (treeSize >= fromSize) // It has to be an upcast { if (toSize < fromSize) { - // Special case: "TYP_INT <- %SMALL_INT% <- TYP_INT" + // Special case: "TYP_INT <- %SMALL_(U)INT% <- TYP_INT" assert(varTypeIsSmall(cast->CastToType()) && (fromSize == 4)); toSize = 4; } assert(toSize >= 4); // LSH(TYP_LONG <- TYP_INT, CNS) => 1..31 - // LSH(TYP_INT <- TYP_SHORT <- TYP_INT, CNS) => 1..15 - // LSH(TYP_INT <- TYP_BYTE <- TYP_INT, CNS) => 1..7 + // LSH(TYP_INT <- TYP_(U)SHORT <- TYP_INT, CNS) => 1..15 + // LSH(TYP_INT <- TYP_(U)BYTE <- TYP_INT, CNS) => 1..7 if ((UINT32)cns->IconValue() < (toSize * 4)) { // cns is small and should already be contained at this point From 261108c10e336693717c54339f985ababe818956 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Mon, 1 Nov 2021 02:21:40 +0300 Subject: [PATCH 06/12] Address feedback & add tests --- src/coreclr/jit/codegenarmarch.cpp | 40 +- src/coreclr/jit/lower.cpp | 38 +- .../opt/InstructionCombining/UbfizSbfiz.cs | 722 ++++++++++++++++++ .../InstructionCombining/UbfizSbfiz.csproj | 12 + 4 files changed, 765 insertions(+), 47 deletions(-) create mode 100644 src/tests/JIT/opt/InstructionCombining/UbfizSbfiz.cs create mode 100644 src/tests/JIT/opt/InstructionCombining/UbfizSbfiz.csproj diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index a57c208d48df2..a70ad50ea3380 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -1614,8 +1614,9 @@ void CodeGen::genCodeForShift(GenTree* tree) genTreeOps oper = tree->OperGet(); instruction ins = genGetInsForOper(oper, targetType); emitAttr size = emitActualTypeSize(tree); + regNumber dstReg = tree->GetRegNum(); - assert(tree->GetRegNum() != REG_NA); + assert(dstReg != REG_NA); genConsumeOperands(tree->AsOp()); @@ -1623,7 +1624,7 @@ void CodeGen::genCodeForShift(GenTree* tree) GenTree* shiftBy = tree->gtGetOp2(); if (!shiftBy->IsCnsIntOrI()) { - GetEmitter()->emitIns_R_R_R(ins, size, tree->GetRegNum(), operand->GetRegNum(), shiftBy->GetRegNum()); + GetEmitter()->emitIns_R_R_R(ins, size, dstReg, operand->GetRegNum(), shiftBy->GetRegNum()); } else { @@ -1634,30 +1635,25 @@ void CodeGen::genCodeForShift(GenTree* tree) // Check if we can recognize ubfiz/sbfiz idiom in LSH(CAST(X), CNS) pattern if (tree->gtGetOp1()->OperIs(GT_CAST) && tree->gtGetOp1()->isContained()) { - GenTreeCast* cast = tree->gtGetOp1()->AsCast(); - int width = 0; - if (varTypeIsSmall(cast->CastToType())) - { - // TYP_INT <- %SMALL_(U)INT% <- TYP_INT - width = (int)genTypeSize(cast->CastToType()) * 8; - assert((width == 16) || (width == 8)); - assert((cast->CastFromType() == TYP_INT) && cast->TypeIs(TYP_INT)); - } - else - { - // TYP_LONG <- TYP_INT - // TYP_LONG <- TYP_UINT <- TYP_INT - // TYP_LONG <- TYP_ULONG <- TYP_INT - width = 32; //(int)genTypeSize(cast->CastFromType()) * 8; - assert((cast->CastFromType() == TYP_INT) && cast->TypeIs(TYP_LONG)); - } - GetEmitter()->emitIns_R_R_I_I(cast->IsUnsigned() ? INS_ubfiz : INS_sbfiz, size, tree->GetRegNum(), - cast->CastOp()->GetRegNum(), (int)shiftByImm, width); + GenTreeCast* cast = tree->gtGetOp1()->AsCast(); + GenTree* castOp = cast->CastOp(); + + unsigned srcBits = varTypeIsSmall(cast->CastToType()) ? genTypeSize(cast->CastToType()) * BITS_PER_BYTE + : genTypeSize(castOp) * BITS_PER_BYTE; + unsigned dstBits = genTypeSize(cast) * BITS_PER_BYTE; + + assert(srcBits < dstBits); + assert((shiftByImm > 0) && (shiftByImm < srcBits)); + assert(shiftBy->isContained()); + + const bool isUnsigned = cast->IsUnsigned() || varTypeIsUnsigned(cast->CastToType()); + GetEmitter()->emitIns_R_R_I_I(isUnsigned ? INS_ubfiz : INS_sbfiz, size, dstReg, castOp->GetRegNum(), + (int)shiftByImm, (int)srcBits); } else #endif { - GetEmitter()->emitIns_R_R_I(ins, size, tree->GetRegNum(), operand->GetRegNum(), shiftByImm); + GetEmitter()->emitIns_R_R_I(ins, size, dstReg, operand->GetRegNum(), shiftByImm); } } diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index 2689470a42a0f..32327ff9282e4 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -5755,7 +5755,7 @@ void Lowering::LowerShift(GenTreeOp* shift) ContainCheckShiftRotate(shift); #ifdef TARGET_ARM64 - // Check if we can recognize ubfiz/sbfiz idiom in LSH(CAST(X), CNS) pattern + // Try to recognize ubfiz/sbfiz idiom in LSH(CAST(X), CNS) tree if (comp->opts.OptimizationEnabled() && shift->OperIs(GT_LSH) && shift->gtGetOp1()->OperIs(GT_CAST) && shift->gtGetOp2()->IsCnsIntOrI() && !shift->isContained() && varTypeIsIntegral(shift)) { @@ -5763,36 +5763,24 @@ void Lowering::LowerShift(GenTreeOp* shift) GenTreeCast* cast = shift->gtGetOp1()->AsCast(); if (!cast->isContained() && !cast->IsRegOptional() && !cast->gtOverflow() && - // CastOp should not be a small int, e.g. CAST(IND(ByteTarget)) + // Smaller CastOp is most likely an IND(X) node which is lowered to a zero-extend load cast->CastOp()->TypeIs(TYP_LONG, TYP_INT)) { - // Cast can be either "TYP_LONG <- TYP_INT" or "TYP_INT <- %SMALL_INT% <- TYP_INT" - UINT32 treeSize = genTypeSize(cast); - UINT32 fromSize = genTypeSize(cast->CastFromType()); - UINT32 toSize = genTypeSize(cast->CastToType()); + // Cast is either "TYP_LONG <- TYP_INT" or "TYP_INT <- %SMALL_INT% <- TYP_INT" (signed or unsigned) + + unsigned srcBits = varTypeIsSmall(cast->CastToType()) ? genTypeSize(cast->CastToType()) * BITS_PER_BYTE + : genTypeSize(cast->CastOp()) * BITS_PER_BYTE; + unsigned dstBits = genTypeSize(cast) * BITS_PER_BYTE; assert(!cast->CastOp()->isContained()); - if (treeSize >= fromSize) // It has to be an upcast + // It has to be an upcast and CNS must be in [1..srcBits) range + if ((srcBits < dstBits) && ((UINT32)cns->IconValue() < srcBits)) { - if (toSize < fromSize) - { - // Special case: "TYP_INT <- %SMALL_(U)INT% <- TYP_INT" - assert(varTypeIsSmall(cast->CastToType()) && (fromSize == 4)); - toSize = 4; - } - assert(toSize >= 4); - - // LSH(TYP_LONG <- TYP_INT, CNS) => 1..31 - // LSH(TYP_INT <- TYP_(U)SHORT <- TYP_INT, CNS) => 1..15 - // LSH(TYP_INT <- TYP_(U)BYTE <- TYP_INT, CNS) => 1..7 - if ((UINT32)cns->IconValue() < (toSize * 4)) - { - // cns is small and should already be contained at this point - assert(cns->isContained()); - JITDUMP("Recognized ubfix/sbfix pattern in LSH(CAST, CNS), marking CAST node as contained."); - MakeSrcContained(shift, cast); - } + // cns is small and should already be contained at this point + assert(cns->isContained()); + JITDUMP("Recognized ubfix/sbfix pattern in LSH(CAST, CNS), marking CAST node as contained."); + MakeSrcContained(shift, cast); } } } diff --git a/src/tests/JIT/opt/InstructionCombining/UbfizSbfiz.cs b/src/tests/JIT/opt/InstructionCombining/UbfizSbfiz.cs new file mode 100644 index 0000000000000..33cba49a0ab4a --- /dev/null +++ b/src/tests/JIT/opt/InstructionCombining/UbfizSbfiz.cs @@ -0,0 +1,722 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Runtime.CompilerServices; + +public class Program +{ + public const int ShiftBy = 5; + + [MethodImpl(MethodImplOptions.NoInlining)] + public static T ToVar(T t) => t; + + public static void AssertTrue(bool cond, [CallerLineNumber] int line = 0) + { + if (!cond) + throw new InvalidOperationException($"Test failed at line {line}."); + } + + // Tests for https://github.com/dotnet/runtime/pull/61045 optimization + public static int Main() + { + unchecked + { + long[] testData = + { + -1, -2, -3, -8, -128, -129, -254, -255, -256, + 0, 1, 2, 3, 8, 128, 129, 254, 255, 256, + short.MinValue + 1, short.MinValue, short.MinValue + 1, + short.MaxValue + 1, short.MaxValue, short.MaxValue + 1, + int.MinValue + 1, int.MinValue, int.MinValue + 1, + int.MaxValue + 1, int.MaxValue, int.MaxValue + 1, + long.MinValue + 1, long.MinValue, long.MinValue + 1, + long.MaxValue + 1, long.MaxValue, long.MaxValue + 1, + ushort.MaxValue, uint.MaxValue, (long)ulong.MaxValue + }; + + foreach (long t in testData) + { + AssertTrue(Tests_byte.Test_byte_to_byte((byte)t)); + AssertTrue(Tests_byte.Test_byte_to_sbyte((byte)t)); + AssertTrue(Tests_byte.Test_byte_to_ushort((byte)t)); + AssertTrue(Tests_byte.Test_byte_to_short((byte)t)); + AssertTrue(Tests_byte.Test_byte_to_uint((byte)t)); + AssertTrue(Tests_byte.Test_byte_to_int((byte)t)); + AssertTrue(Tests_byte.Test_byte_to_ulong((byte)t)); + AssertTrue(Tests_byte.Test_byte_to_long((byte)t)); + + AssertTrue(Tests_sbyte.Test_sbyte_to_byte((sbyte)t)); + AssertTrue(Tests_sbyte.Test_sbyte_to_sbyte((sbyte)t)); + AssertTrue(Tests_sbyte.Test_sbyte_to_ushort((sbyte)t)); + AssertTrue(Tests_sbyte.Test_sbyte_to_short((sbyte)t)); + AssertTrue(Tests_sbyte.Test_sbyte_to_uint((sbyte)t)); + AssertTrue(Tests_sbyte.Test_sbyte_to_int((sbyte)t)); + AssertTrue(Tests_sbyte.Test_sbyte_to_ulong((sbyte)t)); + AssertTrue(Tests_sbyte.Test_sbyte_to_long((sbyte)t)); + + AssertTrue(Tests_ushort.Test_ushort_to_byte((ushort)t)); + AssertTrue(Tests_ushort.Test_ushort_to_sbyte((ushort)t)); + AssertTrue(Tests_ushort.Test_ushort_to_ushort((ushort)t)); + AssertTrue(Tests_ushort.Test_ushort_to_short((ushort)t)); + AssertTrue(Tests_ushort.Test_ushort_to_uint((ushort)t)); + AssertTrue(Tests_ushort.Test_ushort_to_int((ushort)t)); + AssertTrue(Tests_ushort.Test_ushort_to_ulong((ushort)t)); + AssertTrue(Tests_ushort.Test_ushort_to_long((ushort)t)); + + AssertTrue(Tests_short.Test_short_to_byte((short)t)); + AssertTrue(Tests_short.Test_short_to_sbyte((short)t)); + AssertTrue(Tests_short.Test_short_to_ushort((short)t)); + AssertTrue(Tests_short.Test_short_to_short((short)t)); + AssertTrue(Tests_short.Test_short_to_uint((short)t)); + AssertTrue(Tests_short.Test_short_to_int((short)t)); + AssertTrue(Tests_short.Test_short_to_ulong((short)t)); + AssertTrue(Tests_short.Test_short_to_long((short)t)); + + AssertTrue(Tests_uint.Test_uint_to_byte((uint)t)); + AssertTrue(Tests_uint.Test_uint_to_sbyte((uint)t)); + AssertTrue(Tests_uint.Test_uint_to_ushort((uint)t)); + AssertTrue(Tests_uint.Test_uint_to_short((uint)t)); + AssertTrue(Tests_uint.Test_uint_to_uint((uint)t)); + AssertTrue(Tests_uint.Test_uint_to_int((uint)t)); + AssertTrue(Tests_uint.Test_uint_to_ulong((uint)t)); + AssertTrue(Tests_uint.Test_uint_to_long((uint)t)); + + AssertTrue(Tests_int.Test_int_to_byte((int)t)); + AssertTrue(Tests_int.Test_int_to_sbyte((int)t)); + AssertTrue(Tests_int.Test_int_to_ushort((int)t)); + AssertTrue(Tests_int.Test_int_to_short((int)t)); + AssertTrue(Tests_int.Test_int_to_uint((int)t)); + AssertTrue(Tests_int.Test_int_to_int((int)t)); + AssertTrue(Tests_int.Test_int_to_ulong((int)t)); + AssertTrue(Tests_int.Test_int_to_long((int)t)); + + AssertTrue(Tests_ulong.Test_ulong_to_byte((ulong)t)); + AssertTrue(Tests_ulong.Test_ulong_to_sbyte((ulong)t)); + AssertTrue(Tests_ulong.Test_ulong_to_ushort((ulong)t)); + AssertTrue(Tests_ulong.Test_ulong_to_short((ulong)t)); + AssertTrue(Tests_ulong.Test_ulong_to_uint((ulong)t)); + AssertTrue(Tests_ulong.Test_ulong_to_int((ulong)t)); + AssertTrue(Tests_ulong.Test_ulong_to_ulong((ulong)t)); + AssertTrue(Tests_ulong.Test_ulong_to_long((ulong)t)); + + AssertTrue(Tests_long.Test_long_to_byte(t)); + AssertTrue(Tests_long.Test_long_to_sbyte(t)); + AssertTrue(Tests_long.Test_long_to_ushort(t)); + AssertTrue(Tests_long.Test_long_to_short(t)); + AssertTrue(Tests_long.Test_long_to_uint(t)); + AssertTrue(Tests_long.Test_long_to_int(t)); + AssertTrue(Tests_long.Test_long_to_ulong(t)); + AssertTrue(Tests_long.Test_long_to_long(t)); + } + } + return 100; + } +} + +public class Tests_byte +{ + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_byte_to_byte(byte x) + { + unchecked + { + return (byte)(x << Program.ShiftBy) == + (byte)(Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_byte_to_sbyte(byte x) + { + unchecked + { + return (sbyte)((sbyte)x << Program.ShiftBy) == + (sbyte)((sbyte)Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_byte_to_ushort(byte x) + { + unchecked + { + return (ushort)(x << Program.ShiftBy) == + (ushort)(Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_byte_to_short(byte x) + { + unchecked + { + return (short)(x << Program.ShiftBy) == + (short)(Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_byte_to_uint(byte x) + { + return (uint)x << Program.ShiftBy == + (uint)Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_byte_to_int(byte x) + { + return x << Program.ShiftBy == + Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_byte_to_ulong(byte x) + { + return (ulong)x << Program.ShiftBy == + (ulong)Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_byte_to_long(byte x) + { + return (long)x << Program.ShiftBy == + (long)Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } +} + +public class Tests_sbyte +{ + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_sbyte_to_byte(sbyte x) + { + unchecked + { + return (byte)((byte)x << Program.ShiftBy) == + (byte)((byte)Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_sbyte_to_sbyte(sbyte x) + { + unchecked + { + return (sbyte)(x << Program.ShiftBy) == + (sbyte)(Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_sbyte_to_ushort(sbyte x) + { + unchecked + { + return (ushort)((ushort)x << Program.ShiftBy) == + (ushort)((ushort)Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_sbyte_to_short(sbyte x) + { + unchecked + { + return (short)(x << Program.ShiftBy) == + (short)(Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_sbyte_to_uint(sbyte x) + { + unchecked + { + return (uint)x << Program.ShiftBy == + (uint)Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_sbyte_to_int(sbyte x) + { + return x << Program.ShiftBy == + Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_sbyte_to_ulong(sbyte x) + { + unchecked + { + return (ulong)x << Program.ShiftBy == + (ulong)Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_sbyte_to_long(sbyte x) + { + return (long)x << Program.ShiftBy == + (long)Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } +} + +public class Tests_ushort +{ + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_ushort_to_byte(ushort x) + { + unchecked + { + return (byte)((byte)x << Program.ShiftBy) == + (byte)((byte)Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_ushort_to_sbyte(ushort x) + { + unchecked + { + return (sbyte)((sbyte)x << Program.ShiftBy) == + (sbyte)((sbyte)Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_ushort_to_ushort(ushort x) + { + unchecked + { + return (ushort)(x << Program.ShiftBy) == + (ushort)(Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_ushort_to_short(ushort x) + { + unchecked + { + return (short)((short)x << Program.ShiftBy) == + (short)((short)Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_ushort_to_uint(ushort x) + { + return (uint)x << Program.ShiftBy == + (uint)Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_ushort_to_int(ushort x) + { + return x << Program.ShiftBy == + Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_ushort_to_ulong(ushort x) + { + return (ulong)x << Program.ShiftBy == + (ulong)Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_ushort_to_long(ushort x) + { + return (long)x << Program.ShiftBy == + (long)Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } +} + +public class Tests_short +{ + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_short_to_byte(short x) + { + unchecked + { + return (byte)((byte)x << Program.ShiftBy) == + (byte)((byte)Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_short_to_sbyte(short x) + { + unchecked + { + return (sbyte)((sbyte)x << Program.ShiftBy) == + (sbyte)((sbyte)Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_short_to_ushort(short x) + { + unchecked + { + return (ushort)((ushort)x << Program.ShiftBy) == + (ushort)((ushort)Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_short_to_short(short x) + { + unchecked + { + return (short)(x << Program.ShiftBy) == + (short)(Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_short_to_uint(short x) + { + unchecked + { + return (uint)x << Program.ShiftBy == + (uint)Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_short_to_int(short x) + { + return x << Program.ShiftBy == + Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_short_to_ulong(short x) + { + unchecked + { + return (ulong)x << Program.ShiftBy == + (ulong)Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_short_to_long(short x) + { + return (long)x << Program.ShiftBy == + (long)Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } +} + +public class Tests_uint +{ + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_uint_to_byte(uint x) + { + unchecked + { + return (byte)((byte)x << Program.ShiftBy) == + (byte)((byte)Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_uint_to_sbyte(uint x) + { + unchecked + { + return (sbyte)((sbyte)x << Program.ShiftBy) == + (sbyte)((sbyte)Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_uint_to_ushort(uint x) + { + unchecked + { + return (ushort)((ushort)x << Program.ShiftBy) == + (ushort)((ushort)Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_uint_to_short(uint x) + { + unchecked + { + return (short)((short)x << Program.ShiftBy) == + (short)((short)Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_uint_to_uint(uint x) + { + return x << Program.ShiftBy == + Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_uint_to_int(uint x) + { + unchecked + { + return (int)x << Program.ShiftBy == + (int)Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_uint_to_ulong(uint x) + { + return (ulong)x << Program.ShiftBy == + (ulong)Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_uint_to_long(uint x) + { + return (long)x << Program.ShiftBy == + (long)Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } +} + +public class Tests_int +{ + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_int_to_byte(int x) + { + unchecked + { + return (byte)((byte)x << Program.ShiftBy) == + (byte)((byte)Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_int_to_sbyte(int x) + { + unchecked + { + return (sbyte)((sbyte)x << Program.ShiftBy) == + (sbyte)((sbyte)Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_int_to_ushort(int x) + { + unchecked + { + return (ushort)((ushort)x << Program.ShiftBy) == + (ushort)((ushort)Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_int_to_short(int x) + { + unchecked + { + return (short)((short)x << Program.ShiftBy) == + (short)((short)Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_int_to_uint(int x) + { + unchecked + { + return (uint)x << Program.ShiftBy == + (uint)Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_int_to_int(int x) + { + return x << Program.ShiftBy == + Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_int_to_ulong(int x) + { + unchecked + { + return (ulong)x << Program.ShiftBy == + (ulong)Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_int_to_long(int x) + { + return (long)x << Program.ShiftBy == + (long)Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } +} + +public class Tests_ulong +{ + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_ulong_to_byte(ulong x) + { + unchecked + { + return (byte)((byte)x << Program.ShiftBy) == + (byte)((byte)Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_ulong_to_sbyte(ulong x) + { + unchecked + { + return (sbyte)((sbyte)x << Program.ShiftBy) == + (sbyte)((sbyte)Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_ulong_to_ushort(ulong x) + { + unchecked + { + return (ushort)((ushort)x << Program.ShiftBy) == + (ushort)((ushort)Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_ulong_to_short(ulong x) + { + unchecked + { + return (short)((short)x << Program.ShiftBy) == + (short)((short)Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_ulong_to_uint(ulong x) + { + unchecked + { + return (uint)x << Program.ShiftBy == + (uint)Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_ulong_to_int(ulong x) + { + unchecked + { + return (int)x << Program.ShiftBy == + (int)Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_ulong_to_ulong(ulong x) + { + return x << Program.ShiftBy == + Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_ulong_to_long(ulong x) + { + unchecked + { + return (long)x << Program.ShiftBy == + (long)Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } + } +} + +public class Tests_long +{ + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_long_to_byte(long x) + { + unchecked + { + return (byte)((byte)x << Program.ShiftBy) == + (byte)((byte)Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_long_to_sbyte(long x) + { + unchecked + { + return (sbyte)((sbyte)x << Program.ShiftBy) == + (sbyte)((sbyte)Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_long_to_ushort(long x) + { + unchecked + { + return (ushort)((ushort)x << Program.ShiftBy) == + (ushort)((ushort)Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_long_to_short(long x) + { + unchecked + { + return (short)((short)x << Program.ShiftBy) == + (short)((short)Program.ToVar(x) << Program.ToVar(Program.ShiftBy)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_long_to_uint(long x) + { + unchecked + { + return (uint)x << Program.ShiftBy == + (uint)Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_long_to_int(long x) + { + unchecked + { + return (int)x << Program.ShiftBy == + (int)Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_long_to_ulong(long x) + { + unchecked + { + return (ulong)x << Program.ShiftBy == + (ulong)Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static bool Test_long_to_long(long x) + { + return x << Program.ShiftBy == + Program.ToVar(x) << Program.ToVar(Program.ShiftBy); + } +} \ No newline at end of file diff --git a/src/tests/JIT/opt/InstructionCombining/UbfizSbfiz.csproj b/src/tests/JIT/opt/InstructionCombining/UbfizSbfiz.csproj new file mode 100644 index 0000000000000..ae422e19fd397 --- /dev/null +++ b/src/tests/JIT/opt/InstructionCombining/UbfizSbfiz.csproj @@ -0,0 +1,12 @@ + + + Exe + + + None + True + + + + + From 5b0857e6c9bea59134e586181b3e740ecba2b127 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Mon, 1 Nov 2021 02:50:04 +0300 Subject: [PATCH 07/12] Remove asserts --- src/coreclr/jit/codegenarmarch.cpp | 1 - src/coreclr/jit/lower.cpp | 2 -- 2 files changed, 3 deletions(-) diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index a70ad50ea3380..08313c47308e1 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -1644,7 +1644,6 @@ void CodeGen::genCodeForShift(GenTree* tree) assert(srcBits < dstBits); assert((shiftByImm > 0) && (shiftByImm < srcBits)); - assert(shiftBy->isContained()); const bool isUnsigned = cast->IsUnsigned() || varTypeIsUnsigned(cast->CastToType()); GetEmitter()->emitIns_R_R_I_I(isUnsigned ? INS_ubfiz : INS_sbfiz, size, dstReg, castOp->GetRegNum(), diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index 32327ff9282e4..cfeab1dd9f018 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -5777,8 +5777,6 @@ void Lowering::LowerShift(GenTreeOp* shift) // It has to be an upcast and CNS must be in [1..srcBits) range if ((srcBits < dstBits) && ((UINT32)cns->IconValue() < srcBits)) { - // cns is small and should already be contained at this point - assert(cns->isContained()); JITDUMP("Recognized ubfix/sbfix pattern in LSH(CAST, CNS), marking CAST node as contained."); MakeSrcContained(shift, cast); } From cd2d5a56dddc94d0a344a1a41b12780a91b2f591 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Mon, 1 Nov 2021 16:09:16 +0300 Subject: [PATCH 08/12] Clean up --- src/coreclr/jit/lsraarm64.cpp | 11 +---------- src/coreclr/jit/lsrabuild.cpp | 4 ++++ 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index f6cbaf6424dbb..1c66f74d14584 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -275,16 +275,7 @@ int LinearScan::BuildNode(GenTree* tree) case GT_RSH: case GT_RSZ: case GT_ROR: - if (tree->gtGetOp1()->isContained()) - { - assert(tree->OperIs(GT_LSH) && tree->gtGetOp1()->OperIs(GT_CAST) && tree->gtGetOp2()->IsCnsIntOrI()); - BuildUse(tree->gtGetOp1()->gtGetOp1()); - srcCount = 1; - } - else - { - srcCount = BuildBinaryUses(tree->AsOp()); - } + srcCount = BuildBinaryUses(tree->AsOp()); assert(dstCount == 1); BuildDef(tree); break; diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 38e7683f431bb..9402fc16d18de 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -3071,6 +3071,10 @@ int LinearScan::BuildOperandUses(GenTree* node, regMaskTP candidates) return 1; } #endif // FEATURE_HW_INTRINSICS + if (node->OperIs(GT_CAST)) + { + return BuildOperandUses(node->gtGetOp1(), candidates); + } return 0; } From 1092c2589d13fc4130393b5bd1593255a20cfc66 Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Tue, 2 Nov 2021 12:31:07 +0300 Subject: [PATCH 09/12] Update lower.cpp --- src/coreclr/jit/lower.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index cfeab1dd9f018..50145f25998d2 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -5757,7 +5757,7 @@ void Lowering::LowerShift(GenTreeOp* shift) #ifdef TARGET_ARM64 // Try to recognize ubfiz/sbfiz idiom in LSH(CAST(X), CNS) tree if (comp->opts.OptimizationEnabled() && shift->OperIs(GT_LSH) && shift->gtGetOp1()->OperIs(GT_CAST) && - shift->gtGetOp2()->IsCnsIntOrI() && !shift->isContained() && varTypeIsIntegral(shift)) + shift->gtGetOp2()->IsCnsIntOrI() && !shift->isContained()) { GenTreeIntCon* cns = shift->gtGetOp2()->AsIntCon(); GenTreeCast* cast = shift->gtGetOp1()->AsCast(); From b06180f2f97d6e8d8c92a534fb19e32b01bd5eb5 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Wed, 3 Nov 2021 11:47:38 +0300 Subject: [PATCH 10/12] Introduce GT_BFIZ --- src/coreclr/jit/codegen.h | 1 + src/coreclr/jit/codegenarm64.cpp | 32 ++++++++++++++++++++++++++++++ src/coreclr/jit/codegenarmarch.cpp | 29 +++++---------------------- src/coreclr/jit/codegenlinear.cpp | 2 +- src/coreclr/jit/gtlist.h | 3 +++ src/coreclr/jit/lower.cpp | 7 +++---- src/coreclr/jit/lsraarm64.cpp | 5 +++++ src/coreclr/jit/lsrabuild.cpp | 4 ---- 8 files changed, 50 insertions(+), 33 deletions(-) diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index c6b789f86f3cb..e403b7f7bf176 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -1248,6 +1248,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX void genCodeForJumpTrue(GenTreeOp* jtrue); #ifdef TARGET_ARM64 void genCodeForJumpCompare(GenTreeOp* tree); + void genCodeForBfiz(GenTreeOp* tree); #endif // TARGET_ARM64 #if defined(FEATURE_EH_FUNCLETS) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index d68da1a1e2a9b..c4c5211622114 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -9564,4 +9564,36 @@ void CodeGen::instGen_MemoryBarrier(BarrierKind barrierKind) } } +//------------------------------------------------------------------------ +// genCodeForBfiz: Generates the code sequence for a GenTree node that +// represents a bitfield insert in zero with sign/zero extension. +// +// Arguments: +// tree - the bitfield insert in zero node. +// +void CodeGen::genCodeForBfiz(GenTreeOp* tree) +{ + assert(tree->OperIs(GT_BFIZ)); + + emitAttr size = emitActualTypeSize(tree); + unsigned shiftBy = (unsigned)tree->gtGetOp2()->AsIntCon()->IconValue(); + unsigned shiftByImm = shiftBy & (emitter::getBitWidth(size) - 1); + GenTreeCast* cast = tree->gtGetOp1()->AsCast(); + GenTree* castOp = cast->CastOp(); + genConsumeRegs(castOp); + + unsigned dstBits = genTypeSize(cast) * BITS_PER_BYTE; + unsigned srcBits = varTypeIsSmall(cast->CastToType()) ? genTypeSize(cast->CastToType()) * BITS_PER_BYTE + : genTypeSize(castOp) * BITS_PER_BYTE; + + assert(srcBits < dstBits); + assert((shiftByImm > 0) && (shiftByImm < srcBits)); + + const bool isUnsigned = cast->IsUnsigned() || varTypeIsUnsigned(cast->CastToType()); + GetEmitter()->emitIns_R_R_I_I(isUnsigned ? INS_ubfiz : INS_sbfiz, size, tree->GetRegNum(), castOp->GetRegNum(), + (int)shiftByImm, (int)srcBits); + + genProduceReg(tree); +} + #endif // TARGET_ARM64 diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 70729fc4f3cb4..672ce34efa363 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -312,6 +312,10 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) case GT_SWAP: genCodeForSwap(treeNode->AsOp()); break; + + case GT_BFIZ: + genCodeForBfiz(treeNode->AsOp()); + break; #endif // TARGET_ARM64 case GT_JMP: @@ -1630,30 +1634,7 @@ void CodeGen::genCodeForShift(GenTree* tree) { unsigned immWidth = emitter::getBitWidth(size); // For ARM64, immWidth will be set to 32 or 64 unsigned shiftByImm = (unsigned)shiftBy->AsIntCon()->gtIconVal & (immWidth - 1); - -#ifdef TARGET_ARM64 - // Check if we can recognize ubfiz/sbfiz idiom in LSH(CAST(X), CNS) pattern - if (tree->gtGetOp1()->OperIs(GT_CAST) && tree->gtGetOp1()->isContained()) - { - GenTreeCast* cast = tree->gtGetOp1()->AsCast(); - GenTree* castOp = cast->CastOp(); - - unsigned srcBits = varTypeIsSmall(cast->CastToType()) ? genTypeSize(cast->CastToType()) * BITS_PER_BYTE - : genTypeSize(castOp) * BITS_PER_BYTE; - unsigned dstBits = genTypeSize(cast) * BITS_PER_BYTE; - - assert(srcBits < dstBits); - assert((shiftByImm > 0) && (shiftByImm < srcBits)); - - const bool isUnsigned = cast->IsUnsigned() || varTypeIsUnsigned(cast->CastToType()); - GetEmitter()->emitIns_R_R_I_I(isUnsigned ? INS_ubfiz : INS_sbfiz, size, dstReg, castOp->GetRegNum(), - (int)shiftByImm, (int)srcBits); - } - else -#endif - { - GetEmitter()->emitIns_R_R_I(ins, size, dstReg, operand->GetRegNum(), shiftByImm); - } + GetEmitter()->emitIns_R_R_I(ins, size, dstReg, operand->GetRegNum(), shiftByImm); } genProduceReg(tree); diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index 7c96aa996ff0c..e0f48954c65cd 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -1608,7 +1608,7 @@ void CodeGen::genConsumeRegs(GenTree* tree) } #endif // FEATURE_HW_INTRINSICS #endif // TARGET_XARCH - else if (tree->OperIs(GT_BITCAST, GT_NEG, GT_CAST)) + else if (tree->OperIs(GT_BITCAST, GT_NEG)) { genConsumeRegs(tree->gtGetOp1()); } diff --git a/src/coreclr/jit/gtlist.h b/src/coreclr/jit/gtlist.h index aa6f990e7e721..0bfdc8a8d998e 100644 --- a/src/coreclr/jit/gtlist.h +++ b/src/coreclr/jit/gtlist.h @@ -282,6 +282,9 @@ GTNODE(PHI_ARG , GenTreePhiArg ,0,(GTK_LEAF|GTK_LOCAL)) // phi GTNODE(JMPTABLE , GenTree ,0, (GTK_LEAF|GTK_NOCONTAIN)) // Generates the jump table for switches GTNODE(SWITCH_TABLE , GenTreeOp ,0, (GTK_BINOP|GTK_NOVALUE)) // Jump Table based switch construct +#ifdef TARGET_ARM64 +GTNODE(BFIZ, GenTreeBfiz ,0, GTK_BINOP) // Bitfield Insert in Zero +#endif //----------------------------------------------------------------------------- // Nodes used only within the code generator: diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index 50145f25998d2..c525c59cfa062 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -5767,17 +5767,16 @@ void Lowering::LowerShift(GenTreeOp* shift) cast->CastOp()->TypeIs(TYP_LONG, TYP_INT)) { // Cast is either "TYP_LONG <- TYP_INT" or "TYP_INT <- %SMALL_INT% <- TYP_INT" (signed or unsigned) - + unsigned dstBits = genTypeSize(cast) * BITS_PER_BYTE; unsigned srcBits = varTypeIsSmall(cast->CastToType()) ? genTypeSize(cast->CastToType()) * BITS_PER_BYTE : genTypeSize(cast->CastOp()) * BITS_PER_BYTE; - unsigned dstBits = genTypeSize(cast) * BITS_PER_BYTE; - assert(!cast->CastOp()->isContained()); // It has to be an upcast and CNS must be in [1..srcBits) range if ((srcBits < dstBits) && ((UINT32)cns->IconValue() < srcBits)) { - JITDUMP("Recognized ubfix/sbfix pattern in LSH(CAST, CNS), marking CAST node as contained."); + JITDUMP("Recognized ubfix/sbfix pattern in LSH(CAST, CNS). Changing op to GT_BFIZ"); + shift->ChangeOper(GT_BFIZ); MakeSrcContained(shift, cast); } } diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index b7acad960d45e..d5890f48b03fb 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -279,6 +279,11 @@ int LinearScan::BuildNode(GenTree* tree) BuildDef(tree); break; + case GT_BFIZ: + srcCount = BuildOperandUses(tree->gtGetOp1()->gtGetOp1()); + BuildDef(tree); + break; + case GT_RETURNTRAP: // this just turns into a compare of its child with an int // + a conditional call diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 10097bb4e7632..35ae7eb3564cd 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -3072,10 +3072,6 @@ int LinearScan::BuildOperandUses(GenTree* node, regMaskTP candidates) } #endif // FEATURE_HW_INTRINSICS #ifdef TARGET_ARM64 - if (node->OperIs(GT_CAST)) - { - return BuildOperandUses(node->gtGetOp1(), candidates); - } if (node->OperIs(GT_MUL)) { // Can be contained for MultiplyAdd on arm64 From 544a794a9c7200e2f0547b01415e0bb8442cdc18 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Wed, 3 Nov 2021 11:51:38 +0300 Subject: [PATCH 11/12] add assert --- src/coreclr/jit/lsraarm64.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index d5890f48b03fb..d59c8e913b02e 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -280,6 +280,7 @@ int LinearScan::BuildNode(GenTree* tree) break; case GT_BFIZ: + assert(tree->gtGetOp1()->OperIs(GT_CAST)); srcCount = BuildOperandUses(tree->gtGetOp1()->gtGetOp1()); BuildDef(tree); break; From 5cdf240d4baec7e6aed85faaac7d593f6aebf88f Mon Sep 17 00:00:00 2001 From: EgorBo Date: Wed, 3 Nov 2021 12:47:06 +0300 Subject: [PATCH 12/12] Remove redundant checks --- src/coreclr/jit/codegenarm64.cpp | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index c4c5211622114..ac1c7cc091a24 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -9580,19 +9580,13 @@ void CodeGen::genCodeForBfiz(GenTreeOp* tree) unsigned shiftByImm = shiftBy & (emitter::getBitWidth(size) - 1); GenTreeCast* cast = tree->gtGetOp1()->AsCast(); GenTree* castOp = cast->CastOp(); - genConsumeRegs(castOp); - unsigned dstBits = genTypeSize(cast) * BITS_PER_BYTE; + genConsumeRegs(castOp); unsigned srcBits = varTypeIsSmall(cast->CastToType()) ? genTypeSize(cast->CastToType()) * BITS_PER_BYTE : genTypeSize(castOp) * BITS_PER_BYTE; - - assert(srcBits < dstBits); - assert((shiftByImm > 0) && (shiftByImm < srcBits)); - const bool isUnsigned = cast->IsUnsigned() || varTypeIsUnsigned(cast->CastToType()); GetEmitter()->emitIns_R_R_I_I(isUnsigned ? INS_ubfiz : INS_sbfiz, size, tree->GetRegNum(), castOp->GetRegNum(), (int)shiftByImm, (int)srcBits); - genProduceReg(tree); }