diff --git a/src/coreclr/jit/assertionprop.cpp b/src/coreclr/jit/assertionprop.cpp index b87ba035044ba..e3eac137275ce 100644 --- a/src/coreclr/jit/assertionprop.cpp +++ b/src/coreclr/jit/assertionprop.cpp @@ -158,6 +158,7 @@ bool IntegralRange::Contains(int64_t value) const return {SymbolicIntegerValue::Zero, SymbolicIntegerValue::One}; case GT_ARR_LENGTH: + case GT_MDARR_LENGTH: return {SymbolicIntegerValue::Zero, SymbolicIntegerValue::ArrayLenMax}; case GT_CALL: @@ -2690,8 +2691,10 @@ void Compiler::optAssertionGen(GenTree* tree) break; case GT_ARR_LENGTH: - // An array length is an (always R-value) indirection (but doesn't derive from GenTreeIndir). - assertionInfo = optCreateAssertion(tree->AsArrLen()->ArrRef(), nullptr, OAK_NOT_EQUAL); + case GT_MDARR_LENGTH: + case GT_MDARR_LOWER_BOUND: + // An array meta-data access is an (always R-value) indirection (but doesn't derive from GenTreeIndir). + assertionInfo = optCreateAssertion(tree->AsArrCommon()->ArrRef(), nullptr, OAK_NOT_EQUAL); break; case GT_NULLCHECK: diff --git a/src/coreclr/jit/block.cpp b/src/coreclr/jit/block.cpp index 3972a5c7670c9..b30641d0a26a4 100644 --- a/src/coreclr/jit/block.cpp +++ b/src/coreclr/jit/block.cpp @@ -438,9 +438,9 @@ void BasicBlock::dspFlags() { printf("idxlen "); } - if (bbFlags & BBF_HAS_NEWARRAY) + if (bbFlags & BBF_HAS_MD_IDX_LEN) { - printf("new[] "); + printf("mdidxlen "); } if (bbFlags & BBF_HAS_NEWOBJ) { @@ -512,6 +512,10 @@ void BasicBlock::dspFlags() { printf("align "); } + if (bbFlags & BBF_HAS_MDARRAYREF) + { + printf("mdarr "); + } } /***************************************************************************** diff --git a/src/coreclr/jit/block.h b/src/coreclr/jit/block.h index e9a539a2f35fa..c7430bf1796e9 100644 --- a/src/coreclr/jit/block.h +++ b/src/coreclr/jit/block.h @@ -511,8 +511,8 @@ enum BasicBlockFlags : unsigned __int64 // cases, because the BB occurs in a loop, and we've determined that all // paths in the loop body leading to BB include a call. - BBF_HAS_IDX_LEN = MAKE_BBFLAG(20), // BB contains simple index or length expressions on an array local var. - BBF_HAS_NEWARRAY = MAKE_BBFLAG(21), // BB contains 'new' of an array + BBF_HAS_IDX_LEN = MAKE_BBFLAG(20), // BB contains simple index or length expressions on an SD array local var. + BBF_HAS_MD_IDX_LEN = MAKE_BBFLAG(21), // BB contains simple index, length, or lower bound expressions on an MD array local var. BBF_HAS_NEWOBJ = MAKE_BBFLAG(22), // BB contains 'new' of an object type. #if defined(FEATURE_EH_FUNCLETS) && defined(TARGET_ARM) @@ -552,6 +552,7 @@ enum BasicBlockFlags : unsigned __int64 BBF_TAILCALL_SUCCESSOR = MAKE_BBFLAG(40), // BB has pred that has potential tail call BBF_BACKWARD_JUMP_SOURCE = MAKE_BBFLAG(41), // Block is a source of a backward jump + BBF_HAS_MDARRAYREF = MAKE_BBFLAG(42), // Block has a multi-dimensional array reference // The following are sets of flags. @@ -561,8 +562,8 @@ enum BasicBlockFlags : unsigned __int64 // Flags to update when two blocks are compacted - BBF_COMPACT_UPD = BBF_CHANGED | BBF_GC_SAFE_POINT | BBF_HAS_JMP | BBF_HAS_IDX_LEN | BBF_BACKWARD_JUMP | BBF_HAS_NEWARRAY | \ - BBF_HAS_NEWOBJ | BBF_HAS_NULLCHECK, + BBF_COMPACT_UPD = BBF_CHANGED | BBF_GC_SAFE_POINT | BBF_HAS_JMP | BBF_HAS_IDX_LEN | BBF_HAS_MD_IDX_LEN | BBF_BACKWARD_JUMP | \ + BBF_HAS_NEWOBJ | BBF_HAS_NULLCHECK | BBF_HAS_MDARRAYREF, // Flags a block should not have had before it is split. @@ -577,12 +578,11 @@ enum BasicBlockFlags : unsigned __int64 // Flags gained by the bottom block when a block is split. // Note, this is a conservative guess. - // For example, the bottom block might or might not have BBF_HAS_NEWARRAY or BBF_HAS_NULLCHECK, - // but we assume it has BBF_HAS_NEWARRAY and BBF_HAS_NULLCHECK. + // For example, the bottom block might or might not have BBF_HAS_NULLCHECK, but we assume it has BBF_HAS_NULLCHECK. // TODO: Should BBF_RUN_RARELY be added to BBF_SPLIT_GAINED ? - BBF_SPLIT_GAINED = BBF_DONT_REMOVE | BBF_HAS_JMP | BBF_BACKWARD_JUMP | BBF_HAS_IDX_LEN | BBF_HAS_NEWARRAY | BBF_PROF_WEIGHT | \ - BBF_HAS_NEWOBJ | BBF_KEEP_BBJ_ALWAYS | BBF_CLONED_FINALLY_END | BBF_HAS_NULLCHECK | BBF_HAS_HISTOGRAM_PROFILE, + BBF_SPLIT_GAINED = BBF_DONT_REMOVE | BBF_HAS_JMP | BBF_BACKWARD_JUMP | BBF_HAS_IDX_LEN | BBF_HAS_MD_IDX_LEN | BBF_PROF_WEIGHT | \ + BBF_HAS_NEWOBJ | BBF_KEEP_BBJ_ALWAYS | BBF_CLONED_FINALLY_END | BBF_HAS_NULLCHECK | BBF_HAS_HISTOGRAM_PROFILE | BBF_HAS_MDARRAYREF, }; inline constexpr BasicBlockFlags operator ~(BasicBlockFlags a) diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 41ab1420db07e..adb719f975e8a 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -1495,10 +1495,9 @@ void CodeGen::genCodeForArrIndex(GenTreeArrIndex* arrIndex) regNumber tmpReg = arrIndex->GetSingleTempReg(); assert(tgtReg != tmpReg); - unsigned dim = arrIndex->gtCurrDim; - unsigned rank = arrIndex->gtArrRank; - var_types elemType = arrIndex->gtArrElemType; - unsigned offset; + unsigned dim = arrIndex->gtCurrDim; + unsigned rank = arrIndex->gtArrRank; + unsigned offset; offset = compiler->eeGetMDArrayLowerBoundOffset(rank, dim); emit->emitIns_R_R_I(INS_ldr, EA_4BYTE, tmpReg, arrReg, offset); @@ -1549,10 +1548,9 @@ void CodeGen::genCodeForArrOffset(GenTreeArrOffs* arrOffset) regNumber tmpReg = arrOffset->GetSingleTempReg(); - unsigned dim = arrOffset->gtCurrDim; - unsigned rank = arrOffset->gtArrRank; - var_types elemType = arrOffset->gtArrElemType; - unsigned offset = compiler->eeGetMDArrayLengthOffset(rank, dim); + unsigned dim = arrOffset->gtCurrDim; + unsigned rank = arrOffset->gtArrRank; + unsigned offset = compiler->eeGetMDArrayLengthOffset(rank, dim); // Load tmpReg with the dimension size and evaluate // tgtReg = offsetReg*tmpReg + indexReg. diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index ca050dcf49ff9..d4173c557ac5c 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -4236,6 +4236,8 @@ void CodeGen::genCodeForNullCheck(GenTreeIndir* tree) void CodeGen::genCodeForArrIndex(GenTreeArrIndex* arrIndex) { + assert(!compiler->opts.compJitEarlyExpandMDArrays); + GenTree* arrObj = arrIndex->ArrObj(); GenTree* indexNode = arrIndex->IndexExpr(); @@ -4243,9 +4245,8 @@ void CodeGen::genCodeForArrIndex(GenTreeArrIndex* arrIndex) regNumber indexReg = genConsumeReg(indexNode); regNumber tgtReg = arrIndex->GetRegNum(); - unsigned dim = arrIndex->gtCurrDim; - unsigned rank = arrIndex->gtArrRank; - var_types elemType = arrIndex->gtArrElemType; + unsigned dim = arrIndex->gtCurrDim; + unsigned rank = arrIndex->gtArrRank; noway_assert(tgtReg != REG_NA); @@ -4279,6 +4280,8 @@ void CodeGen::genCodeForArrIndex(GenTreeArrIndex* arrIndex) void CodeGen::genCodeForArrOffset(GenTreeArrOffs* arrOffset) { + assert(!compiler->opts.compJitEarlyExpandMDArrays); + GenTree* offsetNode = arrOffset->gtOffset; GenTree* indexNode = arrOffset->gtIndex; GenTree* arrObj = arrOffset->gtArrObj; @@ -4286,9 +4289,8 @@ void CodeGen::genCodeForArrOffset(GenTreeArrOffs* arrOffset) regNumber tgtReg = arrOffset->GetRegNum(); assert(tgtReg != REG_NA); - unsigned dim = arrOffset->gtCurrDim; - unsigned rank = arrOffset->gtArrRank; - var_types elemType = arrOffset->gtArrElemType; + unsigned dim = arrOffset->gtCurrDim; + unsigned rank = arrOffset->gtArrRank; // First, consume the operands in the correct order. regNumber offsetReg = REG_NA; diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 9ec90da68dd0b..5df953c585b18 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -2820,6 +2820,8 @@ void Compiler::compInitOptions(JitFlags* jitFlags) opts.compJitSaveFpLrWithCalleeSavedRegisters = 0; #endif // defined(TARGET_ARM64) + opts.compJitEarlyExpandMDArrays = (JitConfig.JitEarlyExpandMDArrays() != 0); + #ifdef DEBUG opts.dspInstrs = false; opts.dspLines = false; @@ -2992,6 +2994,18 @@ void Compiler::compInitOptions(JitFlags* jitFlags) { opts.optRepeat = true; } + + // If JitEarlyExpandMDArrays is non-zero, then early MD expansion is enabled. + // If JitEarlyExpandMDArrays is zero, then conditionally enable it for functions specfied by + // JitEarlyExpandMDArraysFilter. + if (JitConfig.JitEarlyExpandMDArrays() == 0) + { + if (JitConfig.JitEarlyExpandMDArraysFilter().contains(info.compMethodName, info.compClassName, + &info.compMethodInfo->args)) + { + opts.compJitEarlyExpandMDArrays = true; + } + } } if (verboseDump) @@ -4839,6 +4853,12 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl fgDebugCheckLinks(); #endif + // Morph multi-dimensional array operations. + // (Consider deferring all array operation morphing, including single-dimensional array ops, + // from global morph to here, so cloning doesn't have to deal with morphed forms.) + // + DoPhase(this, PHASE_MORPH_MDARR, &Compiler::fgMorphArrayOps); + // Create the variable table (and compute variable ref counts) // DoPhase(this, PHASE_MARK_LOCAL_VARS, &Compiler::lvaMarkLocalVars); @@ -9834,7 +9854,7 @@ void cTreeFlags(Compiler* comp, GenTree* tree) #endif if (tree->gtFlags & GTF_IND_NONFAULTING) { - if (tree->OperIsIndirOrArrLength()) + if (tree->OperIsIndirOrArrMetaData()) { chars += printf("[IND_NONFAULTING]"); } diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index f56a4a3730d6b..1812255059d76 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -2606,8 +2606,14 @@ class Compiler GenTreeIndir* gtNewIndexIndir(GenTreeIndexAddr* indexAddr); + void gtAnnotateNewArrLen(GenTree* arrLen, BasicBlock* block); + GenTreeArrLen* gtNewArrLen(var_types typ, GenTree* arrayOp, int lenOffset, BasicBlock* block); + GenTreeMDArr* gtNewMDArrLen(GenTree* arrayOp, unsigned dim, unsigned rank, BasicBlock* block); + + GenTreeMDArr* gtNewMDArrLowerBound(GenTree* arrayOp, unsigned dim, unsigned rank, BasicBlock* block); + GenTreeIndir* gtNewIndir(var_types typ, GenTree* addr); GenTree* gtNewNullCheck(GenTree* addr, BasicBlock* basicBlock); @@ -4540,6 +4546,68 @@ class Compiler bool fgMorphBlockStmt(BasicBlock* block, Statement* stmt DEBUGARG(const char* msg)); + //------------------------------------------------------------------------------------------------------------ + // MorphMDArrayTempCache: a simple cache of compiler temporaries in the local variable table, used to minimize + // the number of locals allocated when doing early multi-dimensional array operation expansion. Two types of + // temps are created and cached (due to the two types of temps needed by the MD array expansion): TYP_INT and + // TYP_REF. `GrabTemp` either returns an available temp from the cache or allocates a new temp and returns it + // after adding it to the cache. `Reset` makes all the temps in the cache available for subsequent re-use. + // + class MorphMDArrayTempCache + { + private: + class TempList + { + public: + TempList(Compiler* compiler) + : m_compiler(compiler), m_first(nullptr), m_insertPtr(&m_first), m_nextAvail(nullptr) + { + } + + unsigned GetTemp(); + + void Reset() + { + m_nextAvail = m_first; + } + + private: + struct Node + { + Node(unsigned tmp) : next(nullptr), tmp(tmp) + { + } + + Node* next; + unsigned tmp; + }; + + Compiler* m_compiler; + Node* m_first; + Node** m_insertPtr; + Node* m_nextAvail; + }; + + TempList intTemps; // Temps for genActualType() == TYP_INT + TempList refTemps; // Temps for TYP_REF + + public: + MorphMDArrayTempCache(Compiler* compiler) : intTemps(compiler), refTemps(compiler) + { + } + + unsigned GrabTemp(var_types type); + + void Reset() + { + intTemps.Reset(); + refTemps.Reset(); + } + }; + + bool fgMorphArrayOpsStmt(MorphMDArrayTempCache* pTempCache, BasicBlock* block, Statement* stmt); + PhaseStatus fgMorphArrayOps(); + void fgSetOptions(); #ifdef DEBUG @@ -6750,19 +6818,25 @@ class Compiler } }; -#define OMF_HAS_NEWARRAY 0x00000001 // Method contains 'new' of an array -#define OMF_HAS_NEWOBJ 0x00000002 // Method contains 'new' of an object type. -#define OMF_HAS_ARRAYREF 0x00000004 // Method contains array element loads or stores. -#define OMF_HAS_NULLCHECK 0x00000008 // Method contains null check. -#define OMF_HAS_FATPOINTER 0x00000010 // Method contains call, that needs fat pointer transformation. -#define OMF_HAS_OBJSTACKALLOC 0x00000020 // Method contains an object allocated on the stack. -#define OMF_HAS_GUARDEDDEVIRT 0x00000040 // Method contains guarded devirtualization candidate -#define OMF_HAS_EXPRUNTIMELOOKUP 0x00000080 // Method contains a runtime lookup to an expandable dictionary. -#define OMF_HAS_PATCHPOINT 0x00000100 // Method contains patchpoints -#define OMF_NEEDS_GCPOLLS 0x00000200 // Method needs GC polls -#define OMF_HAS_FROZEN_STRING 0x00000400 // Method has a frozen string (REF constant int), currently only on NativeAOT. +// clang-format off + +#define OMF_HAS_NEWARRAY 0x00000001 // Method contains 'new' of an SD array +#define OMF_HAS_NEWOBJ 0x00000002 // Method contains 'new' of an object type. +#define OMF_HAS_ARRAYREF 0x00000004 // Method contains array element loads or stores. +#define OMF_HAS_NULLCHECK 0x00000008 // Method contains null check. +#define OMF_HAS_FATPOINTER 0x00000010 // Method contains call, that needs fat pointer transformation. +#define OMF_HAS_OBJSTACKALLOC 0x00000020 // Method contains an object allocated on the stack. +#define OMF_HAS_GUARDEDDEVIRT 0x00000040 // Method contains guarded devirtualization candidate +#define OMF_HAS_EXPRUNTIMELOOKUP 0x00000080 // Method contains a runtime lookup to an expandable dictionary. +#define OMF_HAS_PATCHPOINT 0x00000100 // Method contains patchpoints +#define OMF_NEEDS_GCPOLLS 0x00000200 // Method needs GC polls +#define OMF_HAS_FROZEN_STRING 0x00000400 // Method has a frozen string (REF constant int), currently only on NativeAOT. #define OMF_HAS_PARTIAL_COMPILATION_PATCHPOINT 0x00000800 // Method contains partial compilation patchpoints -#define OMF_HAS_TAILCALL_SUCCESSOR 0x00001000 // Method has potential tail call in a non BBJ_RETURN block +#define OMF_HAS_TAILCALL_SUCCESSOR 0x00001000 // Method has potential tail call in a non BBJ_RETURN block +#define OMF_HAS_MDNEWARRAY 0x00002000 // Method contains 'new' of an MD array +#define OMF_HAS_MDARRAYREF 0x00004000 // Method contains multi-dimensional instrinsic array element loads or stores. + + // clang-format on bool doesMethodHaveFatPointer() { @@ -9232,6 +9306,10 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX static const bool compUseSoftFP = false; #endif // ARM_SOFTFP #endif // CONFIGURABLE_ARM_ABI + + // Use early multi-dimensional array operator expansion (expand after loop optimizations; before lowering). + bool compJitEarlyExpandMDArrays; + } opts; static bool s_pAltJitExcludeAssembliesListInitialized; @@ -10724,6 +10802,8 @@ class GenTreeVisitor case GT_COPY: case GT_RELOAD: case GT_ARR_LENGTH: + case GT_MDARR_LENGTH: + case GT_MDARR_LOWER_BOUND: case GT_CAST: case GT_BITCAST: case GT_CKFINITE: diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 30c9b58cb9cd5..08a1c8239fe57 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -1187,23 +1187,36 @@ inline GenTreeIndir* Compiler::gtNewIndexIndir(GenTreeIndexAddr* indexAddr) } //------------------------------------------------------------------------------ -// gtNewArrLen : Helper to create an array length node. +// gtAnnotateNewArrLen : Helper to add flags for new array length nodes. // +// Arguments: +// arrLen - The new GT_ARR_LENGTH or GT_MDARR_LENGTH node +// block - Basic block that will contain the new length node +// +inline void Compiler::gtAnnotateNewArrLen(GenTree* arrLen, BasicBlock* block) +{ + assert(arrLen->OperIs(GT_ARR_LENGTH, GT_MDARR_LENGTH)); + static_assert_no_msg(GTF_ARRLEN_NONFAULTING == GTF_IND_NONFAULTING); + static_assert_no_msg(GTF_MDARRLEN_NONFAULTING == GTF_IND_NONFAULTING); + arrLen->SetIndirExceptionFlags(this); +} + +//------------------------------------------------------------------------------ +// gtNewArrLen : Helper to create an array length node. // // Arguments: -// typ - Type of the node -// arrayOp - Array node +// typ - Type of the node +// arrayOp - Array node // lenOffset - Offset of the length field // block - Basic block that will contain the result // // Return Value: // New GT_ARR_LENGTH node - +// inline GenTreeArrLen* Compiler::gtNewArrLen(var_types typ, GenTree* arrayOp, int lenOffset, BasicBlock* block) { GenTreeArrLen* arrLen = new (this, GT_ARR_LENGTH) GenTreeArrLen(typ, arrayOp, lenOffset); - static_assert_no_msg(GTF_ARRLEN_NONFAULTING == GTF_IND_NONFAULTING); - arrLen->SetIndirExceptionFlags(this); + gtAnnotateNewArrLen(arrLen, block); if (block != nullptr) { block->bbFlags |= BBF_HAS_IDX_LEN; @@ -1212,6 +1225,56 @@ inline GenTreeArrLen* Compiler::gtNewArrLen(var_types typ, GenTree* arrayOp, int return arrLen; } +//------------------------------------------------------------------------------ +// gtNewMDArrLen : Helper to create an MD array length node. +// +// Arguments: +// arrayOp - Array node +// dim - MD array dimension of interest +// rank - MD array rank +// block - Basic block that will contain the result +// +// Return Value: +// New GT_MDARR_LENGTH node +// +inline GenTreeMDArr* Compiler::gtNewMDArrLen(GenTree* arrayOp, unsigned dim, unsigned rank, BasicBlock* block) +{ + GenTreeMDArr* arrLen = new (this, GT_MDARR_LENGTH) GenTreeMDArr(GT_MDARR_LENGTH, arrayOp, dim, rank); + gtAnnotateNewArrLen(arrLen, block); + if (block != nullptr) + { + block->bbFlags |= BBF_HAS_MD_IDX_LEN; + } + assert((optMethodFlags & OMF_HAS_MDARRAYREF) != 0); // Should have been set in the importer. + return arrLen; +} + +//------------------------------------------------------------------------------ +// gtNewMDArrLowerBound : Helper to create an MD array lower bound node. +// +// Arguments: +// arrayOp - Array node +// dim - MD array dimension of interest +// rank - MD array rank +// block - Basic block that will contain the result +// +// Return Value: +// New GT_MDARR_LOWER_BOUND node +// +inline GenTreeMDArr* Compiler::gtNewMDArrLowerBound(GenTree* arrayOp, unsigned dim, unsigned rank, BasicBlock* block) +{ + GenTreeMDArr* arrOp = new (this, GT_MDARR_LOWER_BOUND) GenTreeMDArr(GT_MDARR_LOWER_BOUND, arrayOp, dim, rank); + + static_assert_no_msg(GTF_MDARRLOWERBOUND_NONFAULTING == GTF_IND_NONFAULTING); + arrOp->SetIndirExceptionFlags(this); + if (block != nullptr) + { + block->bbFlags |= BBF_HAS_MD_IDX_LEN; + } + assert((optMethodFlags & OMF_HAS_MDARRAYREF) != 0); // Should have been set in the importer. + return arrOp; +} + //------------------------------------------------------------------------------ // gtNewIndir : Helper to create an indirection node. // @@ -1460,7 +1523,7 @@ inline void GenTree::ChangeOper(genTreeOps oper, ValueNumberUpdate vnUpdate) assert(!OperIsConst(oper)); // use BashToConst() instead GenTreeFlags mask = GTF_COMMON_MASK; - if (this->OperIsIndirOrArrLength() && OperIsIndirOrArrLength(oper)) + if (this->OperIsIndirOrArrMetaData() && OperIsIndirOrArrMetaData(oper)) { mask |= GTF_IND_NONFAULTING; } @@ -1471,7 +1534,7 @@ inline void GenTree::ChangeOper(genTreeOps oper, ValueNumberUpdate vnUpdate) inline void GenTree::ChangeOperUnchecked(genTreeOps oper) { GenTreeFlags mask = GTF_COMMON_MASK; - if (this->OperIsIndirOrArrLength() && OperIsIndirOrArrLength(oper)) + if (this->OperIsIndirOrArrMetaData() && OperIsIndirOrArrMetaData(oper)) { mask |= GTF_IND_NONFAULTING; } @@ -4165,6 +4228,8 @@ void GenTree::VisitOperands(TVisitor visitor) case GT_COPY: case GT_RELOAD: case GT_ARR_LENGTH: + case GT_MDARR_LENGTH: + case GT_MDARR_LOWER_BOUND: case GT_CAST: case GT_BITCAST: case GT_CKFINITE: diff --git a/src/coreclr/jit/compphases.h b/src/coreclr/jit/compphases.h index 455ea7e2242a3..a67f6d52d9fa9 100644 --- a/src/coreclr/jit/compphases.h +++ b/src/coreclr/jit/compphases.h @@ -18,101 +18,103 @@ // (We should never do EndPhase on a phase that has children, only on 'leaf phases.') // "parent" is -1 for leaf phases, otherwise it is the "enumName" of the parent phase. // "measureIR" is true for phases that generate a count of IR nodes during EndPhase when JitConfig.MeasureIR is -// true. +// true. // clang-format off -// enumName stringName shortName hasChildren measureIR -// parent -CompPhaseNameMacro(PHASE_PRE_IMPORT, "Pre-import", "PRE-IMP", false, -1, false) -CompPhaseNameMacro(PHASE_IMPORTATION, "Importation", "IMPORT", false, -1, true) -CompPhaseNameMacro(PHASE_INDXCALL, "Indirect call transform", "INDXCALL", false, -1, true) -CompPhaseNameMacro(PHASE_PATCHPOINTS, "Expand patchpoints", "PPOINT", false, -1, true) -CompPhaseNameMacro(PHASE_POST_IMPORT, "Post-import", "POST-IMP", false, -1, false) -CompPhaseNameMacro(PHASE_IBCPREP, "Profile instrumentation prep", "IBCPREP", false, -1, false) -CompPhaseNameMacro(PHASE_IBCINSTR, "Profile instrumentation", "IBCINSTR", false, -1, false) -CompPhaseNameMacro(PHASE_INCPROFILE, "Profile incorporation", "INCPROF", false, -1, false) -CompPhaseNameMacro(PHASE_MORPH_INIT, "Morph - Init", "MOR-INIT", false, -1, false) -CompPhaseNameMacro(PHASE_MORPH_INLINE, "Morph - Inlining", "MOR-INL", false, -1, true) -CompPhaseNameMacro(PHASE_MORPH_ADD_INTERNAL, "Morph - Add internal blocks", "MOR-ADD", false, -1, true) -CompPhaseNameMacro(PHASE_ALLOCATE_OBJECTS, "Allocate Objects", "ALLOC-OBJ", false, -1, false) -CompPhaseNameMacro(PHASE_EMPTY_TRY, "Remove empty try", "EMPTYTRY", false, -1, false) -CompPhaseNameMacro(PHASE_EMPTY_FINALLY, "Remove empty finally", "EMPTYFIN", false, -1, false) -CompPhaseNameMacro(PHASE_MERGE_FINALLY_CHAINS, "Merge callfinally chains", "MRGCFCHN", false, -1, false) -CompPhaseNameMacro(PHASE_CLONE_FINALLY, "Clone finally", "CLONEFIN", false, -1, false) -CompPhaseNameMacro(PHASE_UPDATE_FINALLY_FLAGS, "Update finally target flags", "UPD-FTF", false, -1, false) -CompPhaseNameMacro(PHASE_COMPUTE_PREDS, "Compute preds", "PREDS", false, -1, false) -CompPhaseNameMacro(PHASE_EARLY_UPDATE_FLOW_GRAPH,"Update flow graph early pass", "UPD-FG-E", false, -1, false) -CompPhaseNameMacro(PHASE_STR_ADRLCL, "Morph - Structs/AddrExp", "MOR-STRAL",false, -1, false) -CompPhaseNameMacro(PHASE_FWD_SUB, "Forward Substitution", "FWD-SUB", false, -1, false) -CompPhaseNameMacro(PHASE_MORPH_IMPBYREF, "Morph - ByRefs", "MOR-BYREF",false, -1, false) -CompPhaseNameMacro(PHASE_PROMOTE_STRUCTS, "Morph - Promote Structs", "PROMOTER" ,false, -1, false) -CompPhaseNameMacro(PHASE_MORPH_GLOBAL, "Morph - Global", "MOR-GLOB", false, -1, false) -CompPhaseNameMacro(PHASE_MORPH_END, "Morph - Finish", "MOR-END", false, -1, true) -CompPhaseNameMacro(PHASE_GS_COOKIE, "GS Cookie", "GS-COOK", false, -1, false) -CompPhaseNameMacro(PHASE_COMPUTE_EDGE_WEIGHTS, "Compute edge weights (1, false)","EDG-WGT", false, -1, false) +// enumName stringName shortName hasChildren +// parent +// measureIR +CompPhaseNameMacro(PHASE_PRE_IMPORT, "Pre-import", "PRE-IMP", false, -1, false) +CompPhaseNameMacro(PHASE_IMPORTATION, "Importation", "IMPORT", false, -1, true) +CompPhaseNameMacro(PHASE_INDXCALL, "Indirect call transform", "INDXCALL", false, -1, true) +CompPhaseNameMacro(PHASE_PATCHPOINTS, "Expand patchpoints", "PPOINT", false, -1, true) +CompPhaseNameMacro(PHASE_POST_IMPORT, "Post-import", "POST-IMP", false, -1, false) +CompPhaseNameMacro(PHASE_IBCPREP, "Profile instrumentation prep", "IBCPREP", false, -1, false) +CompPhaseNameMacro(PHASE_IBCINSTR, "Profile instrumentation", "IBCINSTR", false, -1, false) +CompPhaseNameMacro(PHASE_INCPROFILE, "Profile incorporation", "INCPROF", false, -1, false) +CompPhaseNameMacro(PHASE_MORPH_INIT, "Morph - Init", "MOR-INIT", false, -1, false) +CompPhaseNameMacro(PHASE_MORPH_INLINE, "Morph - Inlining", "MOR-INL", false, -1, true) +CompPhaseNameMacro(PHASE_MORPH_ADD_INTERNAL, "Morph - Add internal blocks", "MOR-ADD", false, -1, true) +CompPhaseNameMacro(PHASE_ALLOCATE_OBJECTS, "Allocate Objects", "ALLOC-OBJ", false, -1, false) +CompPhaseNameMacro(PHASE_EMPTY_TRY, "Remove empty try", "EMPTYTRY", false, -1, false) +CompPhaseNameMacro(PHASE_EMPTY_FINALLY, "Remove empty finally", "EMPTYFIN", false, -1, false) +CompPhaseNameMacro(PHASE_MERGE_FINALLY_CHAINS, "Merge callfinally chains", "MRGCFCHN", false, -1, false) +CompPhaseNameMacro(PHASE_CLONE_FINALLY, "Clone finally", "CLONEFIN", false, -1, false) +CompPhaseNameMacro(PHASE_UPDATE_FINALLY_FLAGS, "Update finally target flags", "UPD-FTF", false, -1, false) +CompPhaseNameMacro(PHASE_COMPUTE_PREDS, "Compute preds", "PREDS", false, -1, false) +CompPhaseNameMacro(PHASE_EARLY_UPDATE_FLOW_GRAPH, "Update flow graph early pass", "UPD-FG-E", false, -1, false) +CompPhaseNameMacro(PHASE_STR_ADRLCL, "Morph - Structs/AddrExp", "MOR-STRAL", false, -1, false) +CompPhaseNameMacro(PHASE_FWD_SUB, "Forward Substitution", "FWD-SUB", false, -1, false) +CompPhaseNameMacro(PHASE_MORPH_IMPBYREF, "Morph - ByRefs", "MOR-BYREF", false, -1, false) +CompPhaseNameMacro(PHASE_PROMOTE_STRUCTS, "Morph - Promote Structs", "PROMOTER" , false, -1, false) +CompPhaseNameMacro(PHASE_MORPH_GLOBAL, "Morph - Global", "MOR-GLOB", false, -1, false) +CompPhaseNameMacro(PHASE_MORPH_END, "Morph - Finish", "MOR-END", false, -1, true) +CompPhaseNameMacro(PHASE_GS_COOKIE, "GS Cookie", "GS-COOK", false, -1, false) +CompPhaseNameMacro(PHASE_COMPUTE_EDGE_WEIGHTS, "Compute edge weights (1, false)","EDG-WGT", false, -1, false) #if defined(FEATURE_EH_FUNCLETS) -CompPhaseNameMacro(PHASE_CREATE_FUNCLETS, "Create EH funclets", "EH-FUNC", false, -1, false) +CompPhaseNameMacro(PHASE_CREATE_FUNCLETS, "Create EH funclets", "EH-FUNC", false, -1, false) #endif // FEATURE_EH_FUNCLETS -CompPhaseNameMacro(PHASE_MERGE_THROWS, "Merge throw blocks", "MRGTHROW", false, -1, false) -CompPhaseNameMacro(PHASE_INVERT_LOOPS, "Invert loops", "LOOP-INV", false, -1, false) -CompPhaseNameMacro(PHASE_OPTIMIZE_FLOW, "Optimize control flow", "OPT-FLOW", false, -1, false) -CompPhaseNameMacro(PHASE_OPTIMIZE_LAYOUT, "Optimize layout", "LAYOUT", false, -1, false) -CompPhaseNameMacro(PHASE_COMPUTE_REACHABILITY, "Compute blocks reachability", "BL_REACH", false, -1, false) -CompPhaseNameMacro(PHASE_SET_BLOCK_WEIGHTS, "Set block weights", "BL-WEIGHTS", false, -1, false) -CompPhaseNameMacro(PHASE_ZERO_INITS, "Redundant zero Inits", "ZERO-INIT", false, -1, false) -CompPhaseNameMacro(PHASE_FIND_LOOPS, "Find loops", "LOOP-FND", false, -1, false) -CompPhaseNameMacro(PHASE_CLONE_LOOPS, "Clone loops", "LP-CLONE", false, -1, false) -CompPhaseNameMacro(PHASE_UNROLL_LOOPS, "Unroll loops", "UNROLL", false, -1, false) -CompPhaseNameMacro(PHASE_CLEAR_LOOP_INFO, "Clear loop info", "LP-CLEAR", false, -1, false) -CompPhaseNameMacro(PHASE_HOIST_LOOP_CODE, "Hoist loop code", "LP-HOIST", false, -1, false) -CompPhaseNameMacro(PHASE_MARK_LOCAL_VARS, "Mark local vars", "MARK-LCL", false, -1, false) -CompPhaseNameMacro(PHASE_OPTIMIZE_ADD_COPIES, "Opt add copies", "OPT-ADD-CP", false, -1, false) -CompPhaseNameMacro(PHASE_OPTIMIZE_BOOLS, "Optimize bools", "OPT-BOOL", false, -1, false) -CompPhaseNameMacro(PHASE_FIND_OPER_ORDER, "Find oper order", "OPER-ORD", false, -1, false) -CompPhaseNameMacro(PHASE_SET_BLOCK_ORDER, "Set block order", "BLK-ORD", false, -1, true) -CompPhaseNameMacro(PHASE_BUILD_SSA, "Build SSA representation", "SSA", true, -1, false) -CompPhaseNameMacro(PHASE_BUILD_SSA_TOPOSORT, "SSA: topological sort", "SSA-SORT", false, PHASE_BUILD_SSA, false) -CompPhaseNameMacro(PHASE_BUILD_SSA_DOMS, "SSA: Doms1", "SSA-DOMS", false, PHASE_BUILD_SSA, false) -CompPhaseNameMacro(PHASE_BUILD_SSA_LIVENESS, "SSA: liveness", "SSA-LIVE", false, PHASE_BUILD_SSA, false) -CompPhaseNameMacro(PHASE_BUILD_SSA_DF, "SSA: DF", "SSA-DF", false, PHASE_BUILD_SSA, false) -CompPhaseNameMacro(PHASE_BUILD_SSA_INSERT_PHIS, "SSA: insert phis", "SSA-PHI", false, PHASE_BUILD_SSA, false) -CompPhaseNameMacro(PHASE_BUILD_SSA_RENAME, "SSA: rename", "SSA-REN", false, PHASE_BUILD_SSA, false) -CompPhaseNameMacro(PHASE_EARLY_PROP, "Early Value Propagation", "ERL-PROP", false, -1, false) -CompPhaseNameMacro(PHASE_VALUE_NUMBER, "Do value numbering", "VAL-NUM", false, -1, false) -CompPhaseNameMacro(PHASE_OPTIMIZE_INDEX_CHECKS, "Optimize index checks", "OPT-CHK", false, -1, false) -CompPhaseNameMacro(PHASE_OPTIMIZE_VALNUM_CSES, "Optimize Valnum CSEs", "OPT-CSE", false, -1, false) -CompPhaseNameMacro(PHASE_VN_COPY_PROP, "VN based copy prop", "CP-PROP", false, -1, false) -CompPhaseNameMacro(PHASE_OPTIMIZE_BRANCHES, "Redundant branch opts", "OPT-BR", false, -1, false) -CompPhaseNameMacro(PHASE_ASSERTION_PROP_MAIN, "Assertion prop", "AST-PROP", false, -1, false) -CompPhaseNameMacro(PHASE_OPT_UPDATE_FLOW_GRAPH, "Update flow graph opt pass", "UPD-FG-O", false, -1, false) -CompPhaseNameMacro(PHASE_COMPUTE_EDGE_WEIGHTS2, "Compute edge weights (2, false)","EDG-WGT2", false, -1, false) -CompPhaseNameMacro(PHASE_INSERT_GC_POLLS, "Insert GC Polls", "GC-POLLS", false, -1, true) -CompPhaseNameMacro(PHASE_DETERMINE_FIRST_COLD_BLOCK, "Determine first cold block", "COLD-BLK", false, -1, true) -CompPhaseNameMacro(PHASE_RATIONALIZE, "Rationalize IR", "RAT", false, -1, false) -CompPhaseNameMacro(PHASE_SIMPLE_LOWERING, "Do 'simple' lowering", "SMP-LWR", false, -1, false) +CompPhaseNameMacro(PHASE_MERGE_THROWS, "Merge throw blocks", "MRGTHROW", false, -1, false) +CompPhaseNameMacro(PHASE_INVERT_LOOPS, "Invert loops", "LOOP-INV", false, -1, false) +CompPhaseNameMacro(PHASE_OPTIMIZE_FLOW, "Optimize control flow", "OPT-FLOW", false, -1, false) +CompPhaseNameMacro(PHASE_OPTIMIZE_LAYOUT, "Optimize layout", "LAYOUT", false, -1, false) +CompPhaseNameMacro(PHASE_COMPUTE_REACHABILITY, "Compute blocks reachability", "BL_REACH", false, -1, false) +CompPhaseNameMacro(PHASE_SET_BLOCK_WEIGHTS, "Set block weights", "BL-WEIGHTS", false, -1, false) +CompPhaseNameMacro(PHASE_ZERO_INITS, "Redundant zero Inits", "ZERO-INIT", false, -1, false) +CompPhaseNameMacro(PHASE_FIND_LOOPS, "Find loops", "LOOP-FND", false, -1, false) +CompPhaseNameMacro(PHASE_CLONE_LOOPS, "Clone loops", "LP-CLONE", false, -1, false) +CompPhaseNameMacro(PHASE_UNROLL_LOOPS, "Unroll loops", "UNROLL", false, -1, false) +CompPhaseNameMacro(PHASE_CLEAR_LOOP_INFO, "Clear loop info", "LP-CLEAR", false, -1, false) +CompPhaseNameMacro(PHASE_MORPH_MDARR, "Morph array ops", "MOR-ARRAY", false, -1, false) +CompPhaseNameMacro(PHASE_HOIST_LOOP_CODE, "Hoist loop code", "LP-HOIST", false, -1, false) +CompPhaseNameMacro(PHASE_MARK_LOCAL_VARS, "Mark local vars", "MARK-LCL", false, -1, false) +CompPhaseNameMacro(PHASE_OPTIMIZE_ADD_COPIES, "Opt add copies", "OPT-ADD-CP", false, -1, false) +CompPhaseNameMacro(PHASE_OPTIMIZE_BOOLS, "Optimize bools", "OPT-BOOL", false, -1, false) +CompPhaseNameMacro(PHASE_FIND_OPER_ORDER, "Find oper order", "OPER-ORD", false, -1, false) +CompPhaseNameMacro(PHASE_SET_BLOCK_ORDER, "Set block order", "BLK-ORD", false, -1, true) +CompPhaseNameMacro(PHASE_BUILD_SSA, "Build SSA representation", "SSA", true, -1, false) +CompPhaseNameMacro(PHASE_BUILD_SSA_TOPOSORT, "SSA: topological sort", "SSA-SORT", false, PHASE_BUILD_SSA, false) +CompPhaseNameMacro(PHASE_BUILD_SSA_DOMS, "SSA: Doms1", "SSA-DOMS", false, PHASE_BUILD_SSA, false) +CompPhaseNameMacro(PHASE_BUILD_SSA_LIVENESS, "SSA: liveness", "SSA-LIVE", false, PHASE_BUILD_SSA, false) +CompPhaseNameMacro(PHASE_BUILD_SSA_DF, "SSA: DF", "SSA-DF", false, PHASE_BUILD_SSA, false) +CompPhaseNameMacro(PHASE_BUILD_SSA_INSERT_PHIS, "SSA: insert phis", "SSA-PHI", false, PHASE_BUILD_SSA, false) +CompPhaseNameMacro(PHASE_BUILD_SSA_RENAME, "SSA: rename", "SSA-REN", false, PHASE_BUILD_SSA, false) +CompPhaseNameMacro(PHASE_EARLY_PROP, "Early Value Propagation", "ERL-PROP", false, -1, false) +CompPhaseNameMacro(PHASE_VALUE_NUMBER, "Do value numbering", "VAL-NUM", false, -1, false) +CompPhaseNameMacro(PHASE_OPTIMIZE_INDEX_CHECKS, "Optimize index checks", "OPT-CHK", false, -1, false) +CompPhaseNameMacro(PHASE_OPTIMIZE_VALNUM_CSES, "Optimize Valnum CSEs", "OPT-CSE", false, -1, false) +CompPhaseNameMacro(PHASE_VN_COPY_PROP, "VN based copy prop", "CP-PROP", false, -1, false) +CompPhaseNameMacro(PHASE_OPTIMIZE_BRANCHES, "Redundant branch opts", "OPT-BR", false, -1, false) +CompPhaseNameMacro(PHASE_ASSERTION_PROP_MAIN, "Assertion prop", "AST-PROP", false, -1, false) +CompPhaseNameMacro(PHASE_OPT_UPDATE_FLOW_GRAPH, "Update flow graph opt pass", "UPD-FG-O", false, -1, false) +CompPhaseNameMacro(PHASE_COMPUTE_EDGE_WEIGHTS2, "Compute edge weights (2, false)","EDG-WGT2", false, -1, false) +CompPhaseNameMacro(PHASE_INSERT_GC_POLLS, "Insert GC Polls", "GC-POLLS", false, -1, true) +CompPhaseNameMacro(PHASE_DETERMINE_FIRST_COLD_BLOCK, "Determine first cold block", "COLD-BLK", false, -1, true) +CompPhaseNameMacro(PHASE_RATIONALIZE, "Rationalize IR", "RAT", false, -1, false) +CompPhaseNameMacro(PHASE_SIMPLE_LOWERING, "Do 'simple' lowering", "SMP-LWR", false, -1, false) -CompPhaseNameMacro(PHASE_LCLVARLIVENESS, "Local var liveness", "LIVENESS", true, -1, false) -CompPhaseNameMacro(PHASE_LCLVARLIVENESS_INIT, "Local var liveness init", "LIV-INIT", false, PHASE_LCLVARLIVENESS, false) -CompPhaseNameMacro(PHASE_LCLVARLIVENESS_PERBLOCK,"Per block local var liveness", "LIV-BLK", false, PHASE_LCLVARLIVENESS, false) -CompPhaseNameMacro(PHASE_LCLVARLIVENESS_INTERBLOCK, "Global local var liveness", "LIV-GLBL", false, PHASE_LCLVARLIVENESS, false) +CompPhaseNameMacro(PHASE_LCLVARLIVENESS, "Local var liveness", "LIVENESS", true, -1, false) +CompPhaseNameMacro(PHASE_LCLVARLIVENESS_INIT, "Local var liveness init", "LIV-INIT", false, PHASE_LCLVARLIVENESS, false) +CompPhaseNameMacro(PHASE_LCLVARLIVENESS_PERBLOCK, "Per block local var liveness", "LIV-BLK", false, PHASE_LCLVARLIVENESS, false) +CompPhaseNameMacro(PHASE_LCLVARLIVENESS_INTERBLOCK, "Global local var liveness", "LIV-GLBL", false, PHASE_LCLVARLIVENESS, false) -CompPhaseNameMacro(PHASE_LOWERING_DECOMP, "Lowering decomposition", "LWR-DEC", false, -1, false) -CompPhaseNameMacro(PHASE_LOWERING, "Lowering nodeinfo", "LWR-INFO", false, -1, true) -CompPhaseNameMacro(PHASE_STACK_LEVEL_SETTER, "Calculate stack level slots", "STK-SET", false, -1, false) -CompPhaseNameMacro(PHASE_LINEAR_SCAN, "Linear scan register alloc", "LSRA", true, -1, true) -CompPhaseNameMacro(PHASE_LINEAR_SCAN_BUILD, "LSRA build intervals", "LSRA-BLD", false, PHASE_LINEAR_SCAN, false) -CompPhaseNameMacro(PHASE_LINEAR_SCAN_ALLOC, "LSRA allocate", "LSRA-ALL", false, PHASE_LINEAR_SCAN, false) -CompPhaseNameMacro(PHASE_LINEAR_SCAN_RESOLVE, "LSRA resolve", "LSRA-RES", false, PHASE_LINEAR_SCAN, false) -CompPhaseNameMacro(PHASE_ALIGN_LOOPS, "Place 'align' instructions", "LOOP-ALIGN", false, -1, false) -CompPhaseNameMacro(PHASE_GENERATE_CODE, "Generate code", "CODEGEN", false, -1, false) -CompPhaseNameMacro(PHASE_EMIT_CODE, "Emit code", "EMIT", false, -1, false) -CompPhaseNameMacro(PHASE_EMIT_GCEH, "Emit GC+EH tables", "EMT-GCEH", false, -1, false) -CompPhaseNameMacro(PHASE_POST_EMIT, "Post-Emit", "POST-EMIT", false, -1, false) +CompPhaseNameMacro(PHASE_LOWERING_DECOMP, "Lowering decomposition", "LWR-DEC", false, -1, false) +CompPhaseNameMacro(PHASE_LOWERING, "Lowering nodeinfo", "LWR-INFO", false, -1, true) +CompPhaseNameMacro(PHASE_STACK_LEVEL_SETTER, "Calculate stack level slots", "STK-SET", false, -1, false) +CompPhaseNameMacro(PHASE_LINEAR_SCAN, "Linear scan register alloc", "LSRA", true, -1, true) +CompPhaseNameMacro(PHASE_LINEAR_SCAN_BUILD, "LSRA build intervals", "LSRA-BLD", false, PHASE_LINEAR_SCAN, false) +CompPhaseNameMacro(PHASE_LINEAR_SCAN_ALLOC, "LSRA allocate", "LSRA-ALL", false, PHASE_LINEAR_SCAN, false) +CompPhaseNameMacro(PHASE_LINEAR_SCAN_RESOLVE, "LSRA resolve", "LSRA-RES", false, PHASE_LINEAR_SCAN, false) +CompPhaseNameMacro(PHASE_ALIGN_LOOPS, "Place 'align' instructions", "LOOP-ALIGN", false, -1, false) +CompPhaseNameMacro(PHASE_GENERATE_CODE, "Generate code", "CODEGEN", false, -1, false) +CompPhaseNameMacro(PHASE_EMIT_CODE, "Emit code", "EMIT", false, -1, false) +CompPhaseNameMacro(PHASE_EMIT_GCEH, "Emit GC+EH tables", "EMT-GCEH", false, -1, false) +CompPhaseNameMacro(PHASE_POST_EMIT, "Post-Emit", "POST-EMIT", false, -1, false) #if MEASURE_CLRAPI_CALLS // The following is a "pseudo-phase" - it aggregates timing info // for calls through ICorJitInfo across all "real" phases. -CompPhaseNameMacro(PHASE_CLR_API, "CLR API calls", "CLR-API", false, -1, false) +CompPhaseNameMacro(PHASE_CLR_API, "CLR API calls", "CLR-API", false, -1, false) #endif // clang-format on diff --git a/src/coreclr/jit/earlyprop.cpp b/src/coreclr/jit/earlyprop.cpp index 2136cf12fa8b4..3adaf9e5fd0f2 100644 --- a/src/coreclr/jit/earlyprop.cpp +++ b/src/coreclr/jit/earlyprop.cpp @@ -17,6 +17,8 @@ bool Compiler::optDoEarlyPropForFunc() { + // TODO-MDArray: bool propMDArrayLen = (optMethodFlags & OMF_HAS_MDNEWARRAY) && (optMethodFlags & + // OMF_HAS_MDARRAYREF); bool propArrayLen = (optMethodFlags & OMF_HAS_NEWARRAY) && (optMethodFlags & OMF_HAS_ARRAYREF); bool propNullCheck = (optMethodFlags & OMF_HAS_NULLCHECK) != 0; return propArrayLen || propNullCheck; @@ -24,6 +26,7 @@ bool Compiler::optDoEarlyPropForFunc() bool Compiler::optDoEarlyPropForBlock(BasicBlock* block) { + // TODO-MDArray: bool bbHasMDArrayRef = (block->bbFlags & BBF_HAS_MD_IDX_LEN) != 0; bool bbHasArrayRef = (block->bbFlags & BBF_HAS_IDX_LEN) != 0; bool bbHasNullCheck = (block->bbFlags & BBF_HAS_NULLCHECK) != 0; return bbHasArrayRef || bbHasNullCheck; @@ -86,7 +89,9 @@ void Compiler::optCheckFlagsAreSet(unsigned methodFlag, // // Null check folding tries to find GT_INDIR(obj + const) that GT_NULLCHECK(obj) can be folded into // and removed. Currently, the algorithm only matches GT_INDIR and GT_NULLCHECK in the same basic block. - +// +// TODO: support GT_MDARR_LENGTH, GT_MDARRAY_LOWER_BOUND +// PhaseStatus Compiler::optEarlyProp() { if (!optDoEarlyPropForFunc()) @@ -170,7 +175,7 @@ GenTree* Compiler::optEarlyPropRewriteTree(GenTree* tree, LocalNumberToNullCheck optPropKind propKind = optPropKind::OPK_INVALID; bool folded = false; - if (tree->OperIsIndirOrArrLength()) + if (tree->OperIsIndirOrArrMetaData()) { // optFoldNullCheck takes care of updating statement info if a null check is removed. folded = optFoldNullCheck(tree, nullCheckMap); @@ -499,15 +504,15 @@ bool Compiler::optFoldNullCheck(GenTree* tree, LocalNumberToNullCheckTreeMap* nu // or // indir(add(x, const2)) // -// (indir is any node for which OperIsIndirOrArrLength() is true.) +// (indir is any node for which OperIsIndirOrArrMetaData() is true.) // // 2. const1 + const2 if sufficiently small. GenTree* Compiler::optFindNullCheckToFold(GenTree* tree, LocalNumberToNullCheckTreeMap* nullCheckMap) { - assert(tree->OperIsIndirOrArrLength()); + assert(tree->OperIsIndirOrArrMetaData()); - GenTree* addr = (tree->OperGet() == GT_ARR_LENGTH) ? tree->AsArrLen()->ArrRef() : tree->AsIndir()->Addr(); + GenTree* addr = tree->GetIndirOrArrMetaDataAddr(); ssize_t offsetValue = 0; diff --git a/src/coreclr/jit/fgdiagnostic.cpp b/src/coreclr/jit/fgdiagnostic.cpp index 55869a29026aa..c573f15543f5d 100644 --- a/src/coreclr/jit/fgdiagnostic.cpp +++ b/src/coreclr/jit/fgdiagnostic.cpp @@ -402,6 +402,22 @@ void Compiler::fgDumpTree(FILE* fgxFile, GenTree* const tree) fgDumpTree(fgxFile, arr); fprintf(fgxFile, ".Length"); } + else if (tree->OperIs(GT_MDARR_LENGTH)) + { + GenTreeMDArr* arrOp = tree->AsMDArr(); + GenTree* arr = arrOp->ArrRef(); + unsigned dim = arrOp->Dim(); + fgDumpTree(fgxFile, arr); + fprintf(fgxFile, ".GetLength(%u)", dim); + } + else if (tree->OperIs(GT_MDARR_LOWER_BOUND)) + { + GenTreeMDArr* arrOp = tree->AsMDArr(); + GenTree* arr = arrOp->ArrRef(); + unsigned dim = arrOp->Dim(); + fgDumpTree(fgxFile, arr); + fprintf(fgxFile, ".GetLowerBound(%u)", dim); + } else { fprintf(fgxFile, "[%s]", GenTree::OpName(tree->OperGet())); @@ -983,7 +999,7 @@ bool Compiler::fgDumpFlowGraph(Phases phase, PhasePosition pos) { fprintf(fgxFile, "\n hot=\"true\""); } - if (block->bbFlags & (BBF_HAS_NEWOBJ | BBF_HAS_NEWARRAY)) + if (block->bbFlags & BBF_HAS_NEWOBJ) { fprintf(fgxFile, "\n callsNew=\"true\""); } diff --git a/src/coreclr/jit/fgopt.cpp b/src/coreclr/jit/fgopt.cpp index 360633d901e93..3548c10cd35f8 100644 --- a/src/coreclr/jit/fgopt.cpp +++ b/src/coreclr/jit/fgopt.cpp @@ -3328,7 +3328,7 @@ bool Compiler::fgBlockEndFavorsTailDuplication(BasicBlock* block, unsigned lclNu { GenTree* const op2 = tree->AsOp()->gtOp2; - if (op2->OperIs(GT_ARR_LENGTH) || op2->OperIsConst() || op2->OperIsCompare()) + if (op2->OperIsArrLength() || op2->OperIsConst() || op2->OperIsCompare()) { return true; } @@ -4108,7 +4108,7 @@ bool Compiler::fgOptimizeBranch(BasicBlock* bJump) gtReverseCond(condTree); // We need to update the following flags of the bJump block if they were set in the bDest block - bJump->bbFlags |= (bDest->bbFlags & (BBF_HAS_NEWOBJ | BBF_HAS_NEWARRAY | BBF_HAS_NULLCHECK | BBF_HAS_IDX_LEN)); + bJump->bbFlags |= (bDest->bbFlags & (BBF_HAS_NEWOBJ | BBF_HAS_NULLCHECK | BBF_HAS_IDX_LEN | BBF_HAS_MD_IDX_LEN)); bJump->bbJumpKind = BBJ_COND; bJump->bbJumpDest = bDest->bbNext; diff --git a/src/coreclr/jit/flowgraph.cpp b/src/coreclr/jit/flowgraph.cpp index f1e74ff10094c..1f52e7731845b 100644 --- a/src/coreclr/jit/flowgraph.cpp +++ b/src/coreclr/jit/flowgraph.cpp @@ -2922,7 +2922,7 @@ PhaseStatus Compiler::fgFindOperOrder() // and computing lvaOutgoingArgSpaceSize. // // Notes: -// Lowers GT_ARR_LENGTH, GT_BOUNDS_CHECK. +// Lowers GT_ARR_LENGTH, GT_MDARR_LENGTH, GT_MDARR_LOWER_BOUND, GT_BOUNDS_CHECK. // // For target ABIs with fixed out args area, computes upper bound on // the size of this area from the calls in the IR. @@ -2948,18 +2948,43 @@ void Compiler::fgSimpleLowering() switch (tree->OperGet()) { case GT_ARR_LENGTH: + case GT_MDARR_LENGTH: + case GT_MDARR_LOWER_BOUND: { - GenTreeArrLen* arrLen = tree->AsArrLen(); - GenTree* arr = arrLen->AsArrLen()->ArrRef(); - GenTree* add; - GenTree* con; + GenTree* arr = tree->AsArrCommon()->ArrRef(); + int lenOffset = 0; - /* Create the expression "*(array_addr + ArrLenOffs)" */ + switch (tree->OperGet()) + { + case GT_ARR_LENGTH: + { + lenOffset = tree->AsArrLen()->ArrLenOffset(); + noway_assert(lenOffset == OFFSETOF__CORINFO_Array__length || + lenOffset == OFFSETOF__CORINFO_String__stringLen); + break; + } + + case GT_MDARR_LENGTH: + lenOffset = (int)eeGetMDArrayLengthOffset(tree->AsMDArr()->Rank(), tree->AsMDArr()->Dim()); + break; + + case GT_MDARR_LOWER_BOUND: + lenOffset = + (int)eeGetMDArrayLowerBoundOffset(tree->AsMDArr()->Rank(), tree->AsMDArr()->Dim()); + break; + + default: + unreached(); + } + + // Create the expression `*(array_addr + lenOffset)` + + GenTree* addr; noway_assert(arr->gtNext == tree); - noway_assert(arrLen->ArrLenOffset() == OFFSETOF__CORINFO_Array__length || - arrLen->ArrLenOffset() == OFFSETOF__CORINFO_String__stringLen); + JITDUMP("Lower %s:\n", GenTree::OpName(tree->OperGet())); + DISPRANGE(LIR::ReadOnlyRange(arr, tree)); if ((arr->gtOper == GT_CNS_INT) && (arr->AsIntCon()->gtIconVal == 0)) { @@ -2968,20 +2993,21 @@ void Compiler::fgSimpleLowering() // an invariant where there is no sum of two constants node, so // let's simply return an indirection of NULL. - add = arr; + addr = arr; } else { - con = gtNewIconNode(arrLen->ArrLenOffset(), TYP_I_IMPL); - add = gtNewOperNode(GT_ADD, TYP_REF, arr, con); - - range.InsertAfter(arr, con, add); + GenTree* con = gtNewIconNode(lenOffset, TYP_I_IMPL); + addr = gtNewOperNode(GT_ADD, TYP_BYREF, arr, con); + range.InsertAfter(arr, con, addr); } // Change to a GT_IND. tree->ChangeOperUnchecked(GT_IND); + tree->AsOp()->gtOp1 = addr; - tree->AsOp()->gtOp1 = add; + JITDUMP("After Lower %s:\n", GenTree::OpName(tree->OperGet())); + DISPRANGE(LIR::ReadOnlyRange(arr, tree)); break; } diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 1ad8b1450f4dd..ae7583871a2bc 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -316,7 +316,8 @@ void GenTree::InitNodeSize() static_assert_no_msg(sizeof(GenTreeQmark) <= TREE_NODE_SZ_LARGE); // *** large node static_assert_no_msg(sizeof(GenTreeIntrinsic) <= TREE_NODE_SZ_LARGE); // *** large node static_assert_no_msg(sizeof(GenTreeIndexAddr) <= TREE_NODE_SZ_LARGE); // *** large node - static_assert_no_msg(sizeof(GenTreeArrLen) <= TREE_NODE_SZ_LARGE); // *** large node + static_assert_no_msg(sizeof(GenTreeArrLen) <= TREE_NODE_SZ_SMALL); + static_assert_no_msg(sizeof(GenTreeMDArr) <= TREE_NODE_SZ_SMALL); static_assert_no_msg(sizeof(GenTreeBoundsChk) <= TREE_NODE_SZ_SMALL); static_assert_no_msg(sizeof(GenTreeArrElem) <= TREE_NODE_SZ_LARGE); // *** large node static_assert_no_msg(sizeof(GenTreeArrIndex) <= TREE_NODE_SZ_LARGE); // *** large node @@ -333,22 +334,22 @@ void GenTree::InitNodeSize() static_assert_no_msg(sizeof(GenTreePhiArg) <= TREE_NODE_SZ_SMALL); static_assert_no_msg(sizeof(GenTreeAllocObj) <= TREE_NODE_SZ_LARGE); // *** large node #ifndef FEATURE_PUT_STRUCT_ARG_STK - static_assert_no_msg(sizeof(GenTreePutArgStk) <= TREE_NODE_SZ_SMALL); + static_assert_no_msg(sizeof(GenTreePutArgStk) <= TREE_NODE_SZ_SMALL); #else // FEATURE_PUT_STRUCT_ARG_STK // TODO-Throughput: This should not need to be a large node. The object info should be // obtained from the child node. - static_assert_no_msg(sizeof(GenTreePutArgStk) <= TREE_NODE_SZ_LARGE); + static_assert_no_msg(sizeof(GenTreePutArgStk) <= TREE_NODE_SZ_LARGE); #if FEATURE_ARG_SPLIT - static_assert_no_msg(sizeof(GenTreePutArgSplit) <= TREE_NODE_SZ_LARGE); + static_assert_no_msg(sizeof(GenTreePutArgSplit) <= TREE_NODE_SZ_LARGE); #endif // FEATURE_ARG_SPLIT #endif // FEATURE_PUT_STRUCT_ARG_STK #ifdef FEATURE_SIMD - static_assert_no_msg(sizeof(GenTreeSIMD) <= TREE_NODE_SZ_SMALL); + static_assert_no_msg(sizeof(GenTreeSIMD) <= TREE_NODE_SZ_SMALL); #endif // FEATURE_SIMD #ifdef FEATURE_HW_INTRINSICS - static_assert_no_msg(sizeof(GenTreeHWIntrinsic) <= TREE_NODE_SZ_SMALL); + static_assert_no_msg(sizeof(GenTreeHWIntrinsic) <= TREE_NODE_SZ_SMALL); #endif // FEATURE_HW_INTRINSICS // clang-format on } @@ -1966,13 +1967,8 @@ GenTree* Compiler::getArrayLengthFromAllocation(GenTree* tree DEBUGARG(BasicBloc default: break; } -#ifdef DEBUG - if ((arrayLength != nullptr) && (block != nullptr)) - { - optCheckFlagsAreSet(OMF_HAS_NEWARRAY, "OMF_HAS_NEWARRAY", BBF_HAS_NEWARRAY, "BBF_HAS_NEWARRAY", tree, - block); - } -#endif + + assert((arrayLength == nullptr) || ((optMethodFlags & OMF_HAS_NEWARRAY) != 0)); } } @@ -2466,6 +2462,14 @@ bool GenTree::Compare(GenTree* op1, GenTree* op2, bool swapOK) return false; } break; + case GT_MDARR_LENGTH: + case GT_MDARR_LOWER_BOUND: + if ((op1->AsMDArr()->Dim() != op2->AsMDArr()->Dim()) || + (op1->AsMDArr()->Rank() != op2->AsMDArr()->Rank())) + { + return false; + } + break; case GT_CAST: if (op1->AsCast()->gtCastType != op2->AsCast()->gtCastType) { @@ -2944,6 +2948,11 @@ unsigned Compiler::gtHashValue(GenTree* tree) case GT_ARR_LENGTH: hash += tree->AsArrLen()->ArrLenOffset(); break; + case GT_MDARR_LENGTH: + case GT_MDARR_LOWER_BOUND: + hash += tree->AsMDArr()->Dim(); + hash += tree->AsMDArr()->Rank(); + break; case GT_CAST: hash ^= tree->AsCast()->gtCastType; break; @@ -4934,9 +4943,11 @@ unsigned Compiler::gtSetEvalOrder(GenTree* tree) break; case GT_ARR_LENGTH: + case GT_MDARR_LENGTH: + case GT_MDARR_LOWER_BOUND: level++; - /* Array Len should be the same as an indirections, which have a costEx of IND_COST_EX */ + // Array meta-data access should be the same as an indirection, which has a costEx of IND_COST_EX. costEx = IND_COST_EX - 1; costSz = 2; break; @@ -5942,6 +5953,8 @@ bool GenTree::TryGetUse(GenTree* operand, GenTree*** pUse) case GT_COPY: case GT_RELOAD: case GT_ARR_LENGTH: + case GT_MDARR_LENGTH: + case GT_MDARR_LOWER_BOUND: case GT_CAST: case GT_BITCAST: case GT_CKFINITE: @@ -6306,8 +6319,8 @@ bool GenTree::OperRequiresCallFlag(Compiler* comp) } //------------------------------------------------------------------------------ -// OperIsImplicitIndir : Check whether the operation contains an implicit -// indirection. +// OperIsImplicitIndir : Check whether the operation contains an implicit indirection. +// // Arguments: // this - a GenTree node // @@ -6317,7 +6330,6 @@ bool GenTree::OperRequiresCallFlag(Compiler* comp) // Note that for the [HW]INTRINSIC nodes we have to examine the // details of the node to determine its result. // - bool GenTree::OperIsImplicitIndir() const { switch (gtOper) @@ -6337,6 +6349,9 @@ bool GenTree::OperIsImplicitIndir() const case GT_ARR_INDEX: case GT_ARR_ELEM: case GT_ARR_OFFSET: + case GT_ARR_LENGTH: + case GT_MDARR_LENGTH: + case GT_MDARR_LOWER_BOUND: return true; case GT_INTRINSIC: return AsIntrinsic()->gtIntrinsicName == NI_System_Object_GetType; @@ -6437,7 +6452,9 @@ ExceptionSetFlags GenTree::OperExceptions(Compiler* comp) return ExceptionSetFlags::None; case GT_ARR_LENGTH: - if (((this->gtFlags & GTF_IND_NONFAULTING) == 0) && comp->fgAddrCouldBeNull(this->AsArrLen()->ArrRef())) + case GT_MDARR_LENGTH: + case GT_MDARR_LOWER_BOUND: + if (((this->gtFlags & GTF_IND_NONFAULTING) == 0) && comp->fgAddrCouldBeNull(this->AsArrCommon()->ArrRef())) { return ExceptionSetFlags::NullReferenceException; } @@ -8426,14 +8443,24 @@ GenTree* Compiler::gtCloneExpr( break; case GT_ARR_LENGTH: - copy = gtNewArrLen(tree->TypeGet(), tree->AsOp()->gtOp1, tree->AsArrLen()->ArrLenOffset(), nullptr); + copy = + gtNewArrLen(tree->TypeGet(), tree->AsArrLen()->ArrRef(), tree->AsArrLen()->ArrLenOffset(), nullptr); + break; + + case GT_MDARR_LENGTH: + copy = + gtNewMDArrLen(tree->AsMDArr()->ArrRef(), tree->AsMDArr()->Dim(), tree->AsMDArr()->Rank(), nullptr); + break; + + case GT_MDARR_LOWER_BOUND: + copy = gtNewMDArrLowerBound(tree->AsMDArr()->ArrRef(), tree->AsMDArr()->Dim(), tree->AsMDArr()->Rank(), + nullptr); break; case GT_ARR_INDEX: copy = new (this, GT_ARR_INDEX) GenTreeArrIndex(tree->TypeGet(), tree->AsArrIndex()->ArrObj(), tree->AsArrIndex()->IndexExpr(), - tree->AsArrIndex()->gtCurrDim, tree->AsArrIndex()->gtArrRank, - tree->AsArrIndex()->gtArrElemType); + tree->AsArrIndex()->gtCurrDim, tree->AsArrIndex()->gtArrRank); break; case GT_QMARK: @@ -8612,19 +8639,18 @@ GenTree* Compiler::gtCloneExpr( } copy = new (this, GT_ARR_ELEM) GenTreeArrElem(arrElem->TypeGet(), gtCloneExpr(arrElem->gtArrObj, addFlags, deepVarNum, deepVarVal), - arrElem->gtArrRank, arrElem->gtArrElemSize, arrElem->gtArrElemType, &inds[0]); + arrElem->gtArrRank, arrElem->gtArrElemSize, &inds[0]); } break; case GT_ARR_OFFSET: { - copy = new (this, GT_ARR_OFFSET) - GenTreeArrOffs(tree->TypeGet(), - gtCloneExpr(tree->AsArrOffs()->gtOffset, addFlags, deepVarNum, deepVarVal), - gtCloneExpr(tree->AsArrOffs()->gtIndex, addFlags, deepVarNum, deepVarVal), - gtCloneExpr(tree->AsArrOffs()->gtArrObj, addFlags, deepVarNum, deepVarVal), - tree->AsArrOffs()->gtCurrDim, tree->AsArrOffs()->gtArrRank, - tree->AsArrOffs()->gtArrElemType); + GenTreeArrOffs* arrOffs = tree->AsArrOffs(); + copy = new (this, GT_ARR_OFFSET) + GenTreeArrOffs(tree->TypeGet(), gtCloneExpr(arrOffs->gtOffset, addFlags, deepVarNum, deepVarVal), + gtCloneExpr(arrOffs->gtIndex, addFlags, deepVarNum, deepVarVal), + gtCloneExpr(arrOffs->gtArrObj, addFlags, deepVarNum, deepVarVal), arrOffs->gtCurrDim, + arrOffs->gtArrRank); } break; @@ -8974,7 +9000,7 @@ void Compiler::gtUpdateNodeOperSideEffects(GenTree* tree) else { tree->gtFlags &= ~GTF_EXCEPT; - if (tree->OperIsIndirOrArrLength()) + if (tree->OperIsIndirOrArrMetaData()) { tree->SetIndirExceptionFlags(this); } @@ -9086,10 +9112,10 @@ Compiler::fgWalkResult Compiler::fgUpdateSideEffectsPost(GenTree** pTree, fgWalk // Update the node's side effects first. fgWalkPost->compiler->gtUpdateNodeOperSideEffectsPost(tree); - // If this node is an indir or array length, and it doesn't have the GTF_EXCEPT bit set, we + // If this node is an indir or array meta-data load, and it doesn't have the GTF_EXCEPT bit set, we // set the GTF_IND_NONFAULTING bit. This needs to be done after all children, and this node, have // been processed. - if (tree->OperIsIndirOrArrLength() && ((tree->gtFlags & GTF_EXCEPT) == 0)) + if (tree->OperIsIndirOrArrMetaData() && ((tree->gtFlags & GTF_EXCEPT) == 0)) { tree->gtFlags |= GTF_IND_NONFAULTING; } @@ -9154,6 +9180,8 @@ bool GenTree::gtRequestSetFlags() { case GT_IND: case GT_ARR_LENGTH: + case GT_MDARR_LENGTH: + case GT_MDARR_LOWER_BOUND: // These will turn into simple load from memory instructions // and we can't force the setting of the flags on load from memory break; @@ -9241,6 +9269,8 @@ GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node) case GT_COPY: case GT_RELOAD: case GT_ARR_LENGTH: + case GT_MDARR_LENGTH: + case GT_MDARR_LOWER_BOUND: case GT_CAST: case GT_BITCAST: case GT_CKFINITE: @@ -9776,7 +9806,7 @@ bool GenTree::Precedes(GenTree* other) // void GenTree::SetIndirExceptionFlags(Compiler* comp) { - assert(OperIsIndirOrArrLength()); + assert(OperIsIndirOrArrMetaData()); if (OperMayThrow(comp)) { @@ -9784,16 +9814,7 @@ void GenTree::SetIndirExceptionFlags(Compiler* comp) return; } - GenTree* addr = nullptr; - if (OperIsIndir()) - { - addr = AsIndir()->Addr(); - } - else - { - assert(gtOper == GT_ARR_LENGTH); - addr = AsArrLen()->ArrRef(); - } + GenTree* addr = GetIndirOrArrMetaDataAddr(); gtFlags |= GTF_IND_NONFAULTING; gtFlags &= ~GTF_EXCEPT; @@ -10639,6 +10660,11 @@ void Compiler::gtDispNode(GenTree* tree, IndentStack* indentStack, _In_ _In_opt_ break; } } + + if (tree->OperIs(GT_MDARR_LENGTH, GT_MDARR_LOWER_BOUND)) + { + printf(" (%u)", tree->AsMDArr()->Dim()); + } } // for tracking down problems in reguse prediction or liveness tracking diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index f3dee9573eb53..45ed0a576300b 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -640,6 +640,10 @@ enum GenTreeFlags : unsigned int GTF_ARRLEN_NONFAULTING = 0x20000000, // GT_ARR_LENGTH -- An array length operation that cannot fault. Same as GT_IND_NONFAULTING. + GTF_MDARRLEN_NONFAULTING = 0x20000000, // GT_MDARR_LENGTH -- An MD array length operation that cannot fault. Same as GT_IND_NONFAULTING. + + GTF_MDARRLOWERBOUND_NONFAULTING = 0x20000000, // GT_MDARR_LOWER_BOUND -- An MD array lower bound operation that cannot fault. Same as GT_IND_NONFAULTING. + GTF_SIMDASHW_OP = 0x80000000, // GT_HWINTRINSIC -- Indicates that the structHandle should be gotten from gtGetStructHandleForSIMD // rather than from gtGetStructHandleForHWSIMD. }; @@ -1582,9 +1586,25 @@ struct GenTree return gtOper == GT_IND || gtOper == GT_STOREIND || gtOper == GT_NULLCHECK || OperIsBlk(gtOper); } - static bool OperIsIndirOrArrLength(genTreeOps gtOper) + static bool OperIsArrLength(genTreeOps gtOper) + { + return (gtOper == GT_ARR_LENGTH) || (gtOper == GT_MDARR_LENGTH); + } + + static bool OperIsMDArr(genTreeOps gtOper) { - return OperIsIndir(gtOper) || (gtOper == GT_ARR_LENGTH); + return (gtOper == GT_MDARR_LENGTH) || (gtOper == GT_MDARR_LOWER_BOUND); + } + + // Is this an access of an SZ array length, MD array length, or MD array lower bounds? + static bool OperIsArrMetaData(genTreeOps gtOper) + { + return (gtOper == GT_ARR_LENGTH) || (gtOper == GT_MDARR_LENGTH) || (gtOper == GT_MDARR_LOWER_BOUND); + } + + static bool OperIsIndirOrArrMetaData(genTreeOps gtOper) + { + return OperIsIndir(gtOper) || OperIsArrMetaData(gtOper); } bool OperIsIndir() const @@ -1592,11 +1612,27 @@ struct GenTree return OperIsIndir(gtOper); } - bool OperIsIndirOrArrLength() const + bool OperIsArrLength() const { - return OperIsIndirOrArrLength(gtOper); + return OperIsArrLength(gtOper); } + bool OperIsMDArr() const + { + return OperIsMDArr(gtOper); + } + + bool OperIsIndirOrArrMetaData() const + { + return OperIsIndirOrArrMetaData(gtOper); + } + + // Helper function to return the array reference of an array length node. + GenTree* GetArrLengthArrRef(); + + // Helper function to return the address of an indir or array meta-data node. + GenTree* GetIndirOrArrMetaDataAddr(); + bool OperIsImplicitIndir() const; static bool OperIsAtomicOp(genTreeOps gtOper) @@ -6422,31 +6458,86 @@ struct GenTreeArrAddr : GenTreeUnOp target_ssize_t* pOffset); }; -/* gtArrLen -- array length (GT_ARR_LENGTH) - GT_ARR_LENGTH is used for "arr.length" */ +// GenTreeArrCommon -- A parent class for GenTreeArrLen, GenTreeMDArr +// (so, accessing array meta-data for either single-dimensional or multi-dimensional arrays). +// Mostly just a convenience to use the ArrRef() accessor. +// +struct GenTreeArrCommon : public GenTreeUnOp +{ + GenTree*& ArrRef() // the array address node + { + return gtOp1; + } -struct GenTreeArrLen : public GenTreeUnOp + GenTreeArrCommon(genTreeOps oper, var_types type, GenTree* arrRef) : GenTreeUnOp(oper, type, arrRef) + { + } + +#if DEBUGGABLE_GENTREE + GenTreeArrCommon() : GenTreeUnOp() + { + } +#endif +}; + +// GenTreeArrLen (GT_ARR_LENGTH) -- single-dimension (SZ) array length. Used for `array.Length`. +// +struct GenTreeArrLen : public GenTreeArrCommon { - GenTree*& ArrRef() + GenTree*& ArrRef() // the array address node { return gtOp1; - } // the array address node + } + private: - int gtArrLenOffset; // constant to add to "gtArrRef" to get the address of the array length. + int gtArrLenOffset; // constant to add to "ArrRef()" to get the address of the array length. public: - inline int ArrLenOffset() + int ArrLenOffset() const { return gtArrLenOffset; } GenTreeArrLen(var_types type, GenTree* arrRef, int lenOffset) - : GenTreeUnOp(GT_ARR_LENGTH, type, arrRef), gtArrLenOffset(lenOffset) + : GenTreeArrCommon(GT_ARR_LENGTH, type, arrRef), gtArrLenOffset(lenOffset) + { + } + +#if DEBUGGABLE_GENTREE + GenTreeArrLen() : GenTreeArrCommon() { } +#endif +}; + +// GenTreeMDArr (GT_MDARR_LENGTH, GT_MDARR_LOWER_BOUND) -- multi-dimension (MD) array length +// or lower bound for a dimension. Used for `array.GetLength(n)`, `array.GetLowerBound(n)`. +// +struct GenTreeMDArr : public GenTreeArrCommon +{ +private: + unsigned gtDim; // array dimension of this array length + unsigned gtRank; // array rank of the array + +public: + unsigned Dim() const + { + return gtDim; + } + + unsigned Rank() const + { + return gtRank; + } + + GenTreeMDArr(genTreeOps oper, GenTree* arrRef, unsigned dim, unsigned rank) + : GenTreeArrCommon(oper, TYP_INT, arrRef), gtDim(dim), gtRank(rank) + { + assert(OperIs(GT_MDARR_LENGTH, GT_MDARR_LOWER_BOUND)); + } #if DEBUGGABLE_GENTREE - GenTreeArrLen() : GenTreeUnOp() + GenTreeMDArr() : GenTreeArrCommon() { } #endif @@ -6483,10 +6574,17 @@ struct GenTreeBoundsChk : public GenTreeOp } #endif - // If this check is against GT_ARR_LENGTH, returns array reference, else "NULL". + // If this check is against GT_ARR_LENGTH or GT_MDARR_LENGTH, returns array reference, else nullptr. GenTree* GetArray() const { - return GetArrayLength()->OperIs(GT_ARR_LENGTH) ? GetArrayLength()->AsArrLen()->ArrRef() : nullptr; + if (GetArrayLength()->OperIsArrLength()) + { + return GetArrayLength()->GetArrLengthArrRef(); + } + else + { + return nullptr; + } } // The index expression. @@ -6518,13 +6616,12 @@ struct GenTreeArrElem : public GenTree // It stores the size of array elements WHEN it can fit // into an "unsigned char". // This has caused VSW 571394. - var_types gtArrElemType; // The array element type // Requires that "inds" is a pointer to an array of "rank" nodes for the indices. - GenTreeArrElem( - var_types type, GenTree* arr, unsigned char rank, unsigned char elemSize, var_types elemType, GenTree** inds) - : GenTree(GT_ARR_ELEM, type), gtArrObj(arr), gtArrRank(rank), gtArrElemSize(elemSize), gtArrElemType(elemType) + GenTreeArrElem(var_types type, GenTree* arr, unsigned char rank, unsigned char elemSize, GenTree** inds) + : GenTree(GT_ARR_ELEM, type), gtArrObj(arr), gtArrRank(rank), gtArrElemSize(elemSize) { + assert(rank <= ArrLen(gtArrInds)); gtFlags |= (arr->gtFlags & GTF_ALL_EFFECT); for (unsigned char i = 0; i < rank; i++) { @@ -6543,7 +6640,7 @@ struct GenTreeArrElem : public GenTree //-------------------------------------------- // // GenTreeArrIndex (gtArrIndex): Expression to bounds-check the index for one dimension of a -// multi-dimensional or non-zero-based array., and compute the effective index +// multi-dimensional or non-zero-based array, and compute the effective index // (i.e. subtracting the lower bound). // // Notes: @@ -6562,7 +6659,7 @@ struct GenTreeArrElem : public GenTree // +--* ArrLen[i, ] (either generalize GT_ARR_LENGTH or add a new node) // +--* // +--* ArrIndex[i, ] -// Which could, for example, be optimized to the following when known to be within bounds: +// which could, for example, be optimized to the following when known to be within bounds: // /--* TempForLowerBoundDim0 // +--* // +--* - (GT_SUB) @@ -6579,20 +6676,11 @@ struct GenTreeArrIndex : public GenTreeOp { return gtOp2; } - unsigned char gtCurrDim; // The current dimension - unsigned char gtArrRank; // Rank of the array - var_types gtArrElemType; // The array element type - - GenTreeArrIndex(var_types type, - GenTree* arrObj, - GenTree* indexExpr, - unsigned char currDim, - unsigned char arrRank, - var_types elemType) - : GenTreeOp(GT_ARR_INDEX, type, arrObj, indexExpr) - , gtCurrDim(currDim) - , gtArrRank(arrRank) - , gtArrElemType(elemType) + unsigned char gtCurrDim; // The current dimension + unsigned char gtArrRank; // Rank of the array + + GenTreeArrIndex(var_types type, GenTree* arrObj, GenTree* indexExpr, unsigned char currDim, unsigned char arrRank) + : GenTreeOp(GT_ARR_INDEX, type, arrObj, indexExpr), gtCurrDim(currDim), gtArrRank(arrRank) { gtFlags |= GTF_EXCEPT; } @@ -6636,31 +6724,24 @@ struct GenTreeArrIndex : public GenTreeOp // struct GenTreeArrOffs : public GenTree { - GenTree* gtOffset; // The accumulated offset for lower dimensions - must be TYP_I_IMPL, and - // will either be a CSE temp, the constant 0, or another GenTreeArrOffs node. - GenTree* gtIndex; // The effective index for the current dimension - must be non-negative - // and can be any expression (though it is likely to be either a GenTreeArrIndex, - // node, a lclVar, or a constant). - GenTree* gtArrObj; // The array object - may be any expression producing an Array reference, - // but is likely to be a lclVar. - unsigned char gtCurrDim; // The current dimension - unsigned char gtArrRank; // Rank of the array - var_types gtArrElemType; // The array element type - - GenTreeArrOffs(var_types type, - GenTree* offset, - GenTree* index, - GenTree* arrObj, - unsigned char currDim, - unsigned char rank, - var_types elemType) + GenTree* gtOffset; // The accumulated offset for lower dimensions - must be TYP_I_IMPL, and + // will either be a CSE temp, the constant 0, or another GenTreeArrOffs node. + GenTree* gtIndex; // The effective index for the current dimension - must be non-negative + // and can be any expression (though it is likely to be either a GenTreeArrIndex, + // node, a lclVar, or a constant). + GenTree* gtArrObj; // The array object - may be any expression producing an Array reference, + // but is likely to be a lclVar. + unsigned char gtCurrDim; // The current dimension + unsigned char gtArrRank; // Rank of the array + + GenTreeArrOffs( + var_types type, GenTree* offset, GenTree* index, GenTree* arrObj, unsigned char currDim, unsigned char rank) : GenTree(GT_ARR_OFFSET, type) , gtOffset(offset) , gtIndex(index) , gtArrObj(arrObj) , gtCurrDim(currDim) , gtArrRank(rank) - , gtArrElemType(elemType) { assert(index->gtFlags & GTF_EXCEPT); gtFlags |= GTF_EXCEPT; @@ -9189,6 +9270,28 @@ inline bool GenTree::isUsedFromSpillTemp() const return false; } +// Helper function to return the array reference of an array length node. +inline GenTree* GenTree::GetArrLengthArrRef() +{ + assert(OperIsArrLength()); + return AsArrCommon()->ArrRef(); +} + +// Helper function to return the address of an indir or array meta-data node. +inline GenTree* GenTree::GetIndirOrArrMetaDataAddr() +{ + assert(OperIsIndirOrArrMetaData()); + + if (OperIsIndir()) + { + return AsIndir()->Addr(); + } + else + { + return AsArrCommon()->ArrRef(); + } +} + /*****************************************************************************/ #ifndef HOST_64BIT diff --git a/src/coreclr/jit/gschecks.cpp b/src/coreclr/jit/gschecks.cpp index 0ce7554d590b3..ced9e3cf97109 100644 --- a/src/coreclr/jit/gschecks.cpp +++ b/src/coreclr/jit/gschecks.cpp @@ -130,6 +130,8 @@ Compiler::fgWalkResult Compiler::gsMarkPtrsAndAssignGroups(GenTree** pTree, fgWa case GT_ARR_ELEM: case GT_ARR_INDEX: case GT_ARR_OFFSET: + case GT_MDARR_LENGTH: + case GT_MDARR_LOWER_BOUND: case GT_FIELD: newState.isUnderIndir = true; diff --git a/src/coreclr/jit/gtlist.h b/src/coreclr/jit/gtlist.h index 53a3618100b9e..9a158c2970b07 100644 --- a/src/coreclr/jit/gtlist.h +++ b/src/coreclr/jit/gtlist.h @@ -90,7 +90,9 @@ GTNODE(STORE_BLK , GenTreeBlk ,0,GTK_BINOP|GTK_EXOP|GTK_NOVALUE) GTNODE(STORE_DYN_BLK , GenTreeStoreDynBlk ,0,GTK_SPECIAL|GTK_NOVALUE) // Dynamically sized block store GTNODE(NULLCHECK , GenTreeIndir ,0,GTK_UNOP|GTK_NOVALUE) // Null checks the source -GTNODE(ARR_LENGTH , GenTreeArrLen ,0,GTK_UNOP|GTK_EXOP) +GTNODE(ARR_LENGTH , GenTreeArrLen ,0,GTK_UNOP|GTK_EXOP) // single-dimension (SZ) array length +GTNODE(MDARR_LENGTH , GenTreeMDArr ,0,GTK_UNOP|GTK_EXOP) // multi-dimension (MD) array length of a specific dimension +GTNODE(MDARR_LOWER_BOUND, GenTreeMDArr ,0,GTK_UNOP|GTK_EXOP) // multi-dimension (MD) array lower bound of a specific dimension GTNODE(FIELD , GenTreeField ,0,GTK_UNOP|GTK_EXOP|DBK_NOTLIR) // Member-field GTNODE(ALLOCOBJ , GenTreeAllocObj ,0,GTK_UNOP|GTK_EXOP|DBK_NOTLIR) // object allocator diff --git a/src/coreclr/jit/gtstructs.h b/src/coreclr/jit/gtstructs.h index 038164cb2f7b8..b14290fd1d129 100644 --- a/src/coreclr/jit/gtstructs.h +++ b/src/coreclr/jit/gtstructs.h @@ -81,7 +81,9 @@ GTSTRUCT_N(MultiOp , GT_SIMD) GTSTRUCT_N(MultiOp , GT_HWINTRINSIC) #endif GTSTRUCT_1(BoundsChk , GT_BOUNDS_CHECK) +GTSTRUCT_3_SPECIAL(ArrCommon , GT_ARR_LENGTH, GT_MDARR_LENGTH, GT_MDARR_LOWER_BOUND) GTSTRUCT_1(ArrLen , GT_ARR_LENGTH) +GTSTRUCT_2(MDArr , GT_MDARR_LENGTH, GT_MDARR_LOWER_BOUND) GTSTRUCT_1(ArrElem , GT_ARR_ELEM) GTSTRUCT_1(ArrOffs , GT_ARR_OFFSET) GTSTRUCT_1(ArrIndex , GT_ARR_INDEX) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 3caad0781ac9d..0988b7c7aeed6 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -6179,29 +6179,46 @@ GenTree* Compiler::impUnsupportedNamedIntrinsic(unsigned helper, } } -/*****************************************************************************/ - +//------------------------------------------------------------------------ +// impArrayAccessIntrinsic: try to replace a multi-dimensional array intrinsics with IR nodes. +// +// Arguments: +// clsHnd - handle for the intrinsic method's class +// sig - signature of the intrinsic method +// memberRef - the token for the intrinsic method +// readonlyCall - true if call has a readonly prefix +// intrinsicName - the intrinsic to expand: one of NI_Array_Address, NI_Array_Get, NI_Array_Set +// +// Return Value: +// The intrinsic expansion, or nullptr if the expansion was not done (and a function call should be made instead). +// GenTree* Compiler::impArrayAccessIntrinsic( CORINFO_CLASS_HANDLE clsHnd, CORINFO_SIG_INFO* sig, int memberRef, bool readonlyCall, NamedIntrinsic intrinsicName) { - /* If we are generating SMALL_CODE, we don't want to use intrinsics for - the following, as it generates fatter code. - */ + assert((intrinsicName == NI_Array_Address) || (intrinsicName == NI_Array_Get) || (intrinsicName == NI_Array_Set)); + // If we are generating SMALL_CODE, we don't want to use intrinsics, as it generates fatter code. if (compCodeOpt() == SMALL_CODE) { + JITDUMP("impArrayAccessIntrinsic: rejecting array intrinsic due to SMALL_CODE\n"); return nullptr; } - /* These intrinsics generate fatter (but faster) code and are only - done if we don't need SMALL_CODE */ - unsigned rank = (intrinsicName == NI_Array_Set) ? (sig->numArgs - 1) : sig->numArgs; - // The rank 1 case is special because it has to handle two array formats - // we will simply not do that case - if (rank > GT_ARR_MAX_RANK || rank <= 1) + // Handle a maximum rank of GT_ARR_MAX_RANK (3). This is an implementation choice (larger ranks are expected + // to be rare) and could be increased. + if (rank > GT_ARR_MAX_RANK) { + JITDUMP("impArrayAccessIntrinsic: rejecting array intrinsic because rank (%d) > GT_ARR_MAX_RANK (%d)\n", rank, + GT_ARR_MAX_RANK); + return nullptr; + } + + // The rank 1 case is special because it has to handle two array formats. We will simply not do that case. + if (rank <= 1) + { + JITDUMP("impArrayAccessIntrinsic: rejecting array intrinsic because rank (%d) <= 1\n", rank); return nullptr; } @@ -6210,7 +6227,7 @@ GenTree* Compiler::impArrayAccessIntrinsic( // For the ref case, we will only be able to inline if the types match // (verifier checks for this, we don't care for the nonverified case and the - // type is final (so we don't need to do the cast) + // type is final (so we don't need to do the cast)) if ((intrinsicName != NI_Array_Get) && !readonlyCall && varTypeIsGC(elemType)) { // Get the call site signature @@ -6245,6 +6262,8 @@ GenTree* Compiler::impArrayAccessIntrinsic( // if it's not final, we can't do the optimization if (!(info.compCompHnd->getClassAttribs(actualElemClsHnd) & CORINFO_FLG_FINAL)) { + JITDUMP("impArrayAccessIntrinsic: rejecting array intrinsic because actualElemClsHnd (%p) is not final\n", + dspPtr(actualElemClsHnd)); return nullptr; } } @@ -6253,7 +6272,6 @@ GenTree* Compiler::impArrayAccessIntrinsic( if (elemType == TYP_STRUCT) { assert(arrElemClsHnd); - arrayElemSize = info.compCompHnd->getClassSize(arrElemClsHnd); } else @@ -6265,6 +6283,8 @@ GenTree* Compiler::impArrayAccessIntrinsic( { // arrayElemSize would be truncated as an unsigned char. // This means the array element is too large. Don't do the optimization. + JITDUMP("impArrayAccessIntrinsic: rejecting array intrinsic because arrayElemSize (%d) is too large\n", + arrayElemSize); return nullptr; } @@ -6273,32 +6293,48 @@ GenTree* Compiler::impArrayAccessIntrinsic( if (intrinsicName == NI_Array_Set) { // Assignment of a struct is more work, and there are more gets than sets. + // TODO-CQ: support SET (`a[i,j,k] = s`) for struct element arrays. if (elemType == TYP_STRUCT) { + JITDUMP("impArrayAccessIntrinsic: rejecting SET array intrinsic because elemType is TYP_STRUCT" + " (implementation limitation)\n", + arrayElemSize); return nullptr; } val = impPopStack().val; - assert(genActualType(elemType) == genActualType(val->gtType) || + assert((genActualType(elemType) == genActualType(val->gtType)) || (elemType == TYP_FLOAT && val->gtType == TYP_DOUBLE) || (elemType == TYP_INT && val->gtType == TYP_BYREF) || (elemType == TYP_DOUBLE && val->gtType == TYP_FLOAT)); } + // Here, we're committed to expanding the intrinsic and creating a GT_ARR_ELEM node. + optMethodFlags |= OMF_HAS_MDARRAYREF; + compCurBB->bbFlags |= BBF_HAS_MDARRAYREF; + noway_assert((unsigned char)GT_ARR_MAX_RANK == GT_ARR_MAX_RANK); GenTree* inds[GT_ARR_MAX_RANK]; for (unsigned k = rank; k > 0; k--) { - inds[k - 1] = impPopStack().val; + // The indices should be converted to `int` type, as they would be if the intrinsic was not expanded. + GenTree* argVal = impPopStack().val; + if (impInlineRoot()->opts.compJitEarlyExpandMDArrays) + { + // This is only enabled when early MD expansion is set because it causes small + // asm diffs (only in some test cases) otherwise. The GT_ARR_ELEM lowering code "accidentally" does + // this cast, but the new code requires it to be explicit. + argVal = impImplicitIorI4Cast(argVal, TYP_INT); + } + inds[k - 1] = argVal; } GenTree* arr = impPopStack().val; assert(arr->gtType == TYP_REF); - GenTree* arrElem = - new (this, GT_ARR_ELEM) GenTreeArrElem(TYP_BYREF, arr, static_cast(rank), - static_cast(arrayElemSize), elemType, &inds[0]); + GenTree* arrElem = new (this, GT_ARR_ELEM) GenTreeArrElem(TYP_BYREF, arr, static_cast(rank), + static_cast(arrayElemSize), &inds[0]); if (intrinsicName != NI_Array_Address) { @@ -8186,8 +8222,8 @@ void Compiler::impImportNewObjArray(CORINFO_RESOLVED_TOKEN* pResolvedToken, CORI node->AsCall()->compileTimeHelperArgumentHandle = (CORINFO_GENERIC_HANDLE)pResolvedToken->hClass; - // Remember that this basic block contains 'new' of a md array - compCurBB->bbFlags |= BBF_HAS_NEWARRAY; + // Remember that this function contains 'new' of a MD array. + optMethodFlags |= OMF_HAS_MDNEWARRAY; impPushOnStack(node, typeInfo(TI_REF, pResolvedToken->hClass)); } @@ -16444,9 +16480,7 @@ void Compiler::impImportBlockCode(BasicBlock* block) op1->AsCall()->compileTimeHelperArgumentHandle = (CORINFO_GENERIC_HANDLE)resolvedToken.hClass; - /* Remember that this basic block contains 'new' of an sd array */ - - block->bbFlags |= BBF_HAS_NEWARRAY; + // Remember that this function contains 'new' of an SD array. optMethodFlags |= OMF_HAS_NEWARRAY; /* Push the result of the call on the stack */ diff --git a/src/coreclr/jit/jitconfigvalues.h b/src/coreclr/jit/jitconfigvalues.h index 9ae04f0505953..94d2fe167848b 100644 --- a/src/coreclr/jit/jitconfigvalues.h +++ b/src/coreclr/jit/jitconfigvalues.h @@ -273,6 +273,15 @@ CONFIG_INTEGER(EnableIncompleteISAClass, W("EnableIncompleteISAClass"), 0) // En #endif // defined(DEBUG) +CONFIG_INTEGER(JitEarlyExpandMDArrays, W("JitEarlyExpandMDArrays"), 1) // Enable early expansion of multi-dimensional + // array access + +#ifdef DEBUG +CONFIG_METHODSET(JitEarlyExpandMDArraysFilter, W("JitEarlyExpandMDArraysFilter")) // Filter functions with early + // expansion of multi-dimensional + // array access +#endif + #if FEATURE_LOOP_ALIGN CONFIG_INTEGER(JitAlignLoops, W("JitAlignLoops"), 1) // If set, align inner loops #else diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index c41145acafeec..1f9a9f0fab687 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -261,12 +261,20 @@ GenTree* Lowering::LowerNode(GenTree* node) #endif // TARGET_XARCH case GT_ARR_ELEM: - return LowerArrElem(node); + assert(!comp->opts.compJitEarlyExpandMDArrays); + return LowerArrElem(node->AsArrElem()); case GT_ARR_OFFSET: + assert(!comp->opts.compJitEarlyExpandMDArrays); ContainCheckArrOffset(node->AsArrOffs()); break; + case GT_MDARR_LENGTH: + case GT_MDARR_LOWER_BOUND: + // Lowered by fgSimpleLowering() + unreached(); + break; + case GT_ROL: case GT_ROR: LowerRotate(node); @@ -6200,40 +6208,39 @@ void Lowering::WidenSIMD12IfNecessary(GenTreeLclVarCommon* node) // // Notes: // This performs the following lowering. We start with a node of the form: -// /--* -// +--* -// +--* -// /--* arrMD&[,] +// /--* ref +// +--* int +// +--* int +// /--* ARR_ELEM[,] byref // // First, we create temps for arrObj if it is not already a lclVar, and for any of the index // expressions that have side-effects. // We then transform the tree into: -// +// +// // /--* // +--* -// /--* ArrIndex[i, ] +// /--* ARR_INDEX[i, ] // +--* -// /--| arrOffs[i, ] +// /--| ARR_OFFSET[i, ] // | +--* // | +--* -// +--* ArrIndex[*,j] +// +--* ARR_INDEX[*,j] // +--* -// /--| arrOffs[*,j] +// /--| ARR_OFFSET[*,j] // +--* lclVar NewTemp // /--* lea (scale = element size, offset = offset of first element) // -// The new stmtExpr may be omitted if the is a lclVar. -// The new stmtExpr may be embedded if the is not the first tree in linear order for -// the statement containing the original arrMD. -// Note that the arrMDOffs is the INDEX of the lea, but is evaluated before the BASE (which is the second +// 1. The new stmtExpr may be omitted if the is a lclVar. +// 2. The new stmtExpr may be embedded if the is not the first tree in linear order for +// the statement containing the original ARR_ELEM. +// 3. Note that the arrMDOffs is the INDEX of the lea, but is evaluated before the BASE (which is the second // reference to NewTemp), because that provides more accurate lifetimes. -// There may be 1, 2 or 3 dimensions, with 1, 2 or 3 arrMDIdx nodes, respectively. +// 4. There may be 1, 2 or 3 dimensions, with 1, 2 or 3 ARR_INDEX nodes, respectively. // -GenTree* Lowering::LowerArrElem(GenTree* node) +GenTree* Lowering::LowerArrElem(GenTreeArrElem* arrElem) { - // This will assert if we don't have an ArrElem node - GenTreeArrElem* arrElem = node->AsArrElem(); - const unsigned char rank = arrElem->gtArrRank; + const unsigned char rank = arrElem->gtArrRank; JITDUMP("Lowering ArrElem\n"); JITDUMP("============\n"); @@ -6276,16 +6283,16 @@ GenTree* Lowering::LowerArrElem(GenTree* node) } // Next comes the GT_ARR_INDEX node. - GenTreeArrIndex* arrMDIdx = new (comp, GT_ARR_INDEX) - GenTreeArrIndex(TYP_INT, idxArrObjNode, indexNode, dim, rank, arrElem->gtArrElemType); + GenTreeArrIndex* arrMDIdx = + new (comp, GT_ARR_INDEX) GenTreeArrIndex(TYP_INT, idxArrObjNode, indexNode, dim, rank); arrMDIdx->gtFlags |= ((idxArrObjNode->gtFlags | indexNode->gtFlags) & GTF_ALL_EFFECT); BlockRange().InsertBefore(insertionPoint, arrMDIdx); GenTree* offsArrObjNode = comp->gtClone(arrObjNode); BlockRange().InsertBefore(insertionPoint, offsArrObjNode); - GenTreeArrOffs* arrOffs = new (comp, GT_ARR_OFFSET) - GenTreeArrOffs(TYP_I_IMPL, prevArrOffs, arrMDIdx, offsArrObjNode, dim, rank, arrElem->gtArrElemType); + GenTreeArrOffs* arrOffs = + new (comp, GT_ARR_OFFSET) GenTreeArrOffs(TYP_I_IMPL, prevArrOffs, arrMDIdx, offsArrObjNode, dim, rank); arrOffs->gtFlags |= ((prevArrOffs->gtFlags | arrMDIdx->gtFlags | offsArrObjNode->gtFlags) & GTF_ALL_EFFECT); BlockRange().InsertBefore(insertionPoint, arrOffs); @@ -6946,7 +6953,7 @@ void Lowering::ContainCheckArrOffset(GenTreeArrOffs* node) // we don't want to generate code for this if (node->gtOffset->IsIntegralConst(0)) { - MakeSrcContained(node, node->AsArrOffs()->gtOffset); + MakeSrcContained(node, node->gtOffset); } } @@ -6959,7 +6966,7 @@ void Lowering::ContainCheckArrOffset(GenTreeArrOffs* node) void Lowering::ContainCheckLclHeap(GenTreeOp* node) { assert(node->OperIs(GT_LCLHEAP)); - GenTree* size = node->AsOp()->gtOp1; + GenTree* size = node->gtOp1; if (size->IsCnsIntOrI()) { MakeSrcContained(node, size); diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h index b65d57398cc79..46e1accf726cb 100644 --- a/src/coreclr/jit/lower.h +++ b/src/coreclr/jit/lower.h @@ -333,7 +333,7 @@ class Lowering final : public Phase void WidenSIMD12IfNecessary(GenTreeLclVarCommon* node); bool CheckMultiRegLclVar(GenTreeLclVar* lclNode, const ReturnTypeDesc* retTypeDesc); void LowerStoreLoc(GenTreeLclVarCommon* tree); - GenTree* LowerArrElem(GenTree* node); + GenTree* LowerArrElem(GenTreeArrElem* arrElem); void LowerRotate(GenTree* tree); void LowerShift(GenTreeOp* shift); #ifdef FEATURE_SIMD diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 07901100d18e1..b5401b568ca7d 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -10982,7 +10982,7 @@ GenTree* Compiler::fgMorphSmpOp(GenTree* tree, MorphAddrContext* mac) DONE_MORPHING_CHILDREN: #endif // !TARGET_64BIT - if (tree->OperIsIndirOrArrLength()) + if (tree->OperIsIndirOrArrMetaData()) { tree->SetIndirExceptionFlags(this); } @@ -17531,3 +17531,423 @@ GenTree* Compiler::fgMorphReduceAddOps(GenTree* tree) return morphed; } + +//------------------------------------------------------------------------ +// Compiler::MorphMDArrayTempCache::TempList::GetTemp: return a local variable number to use as a temporary variable +// in multi-dimensional array operation expansion. +// +// A temp is either re-used from the cache, or allocated and added to the cache. +// +// Returns: +// A local variable temp number. +// +unsigned Compiler::MorphMDArrayTempCache::TempList::GetTemp() +{ + if (m_nextAvail != nullptr) + { + unsigned tmp = m_nextAvail->tmp; + JITDUMP("Reusing temp V%02u\n", tmp); + m_nextAvail = m_nextAvail->next; + return tmp; + } + else + { + unsigned newTmp = m_compiler->lvaGrabTemp(true DEBUGARG("MD array shared temp")); + Node* newNode = new (m_compiler, CMK_Unknown) Node(newTmp); + assert(m_insertPtr != nullptr); + assert(*m_insertPtr == nullptr); + *m_insertPtr = newNode; + m_insertPtr = &newNode->next; + return newTmp; + } +} + +//------------------------------------------------------------------------ +// Compiler::MorphMDArrayTempCache::GrabTemp: return a local variable number to use as a temporary variable +// in multi-dimensional array operation expansion. +// +// Arguments: +// type - type of temp to get +// +// Returns: +// A local variable temp number. +// +unsigned Compiler::MorphMDArrayTempCache::GrabTemp(var_types type) +{ + switch (genActualType(type)) + { + case TYP_INT: + return intTemps.GetTemp(); + case TYP_REF: + return refTemps.GetTemp(); + default: + unreached(); + } +} + +//------------------------------------------------------------------------ +// fgMorphArrayOpsStmt: Tree walk a statement to morph GT_ARR_ELEM. +// +// The nested `MorphMDArrayVisitor::PostOrderVisit()` does the morphing. +// +// See the comment for `fgMorphArrayOps()` for more details of the transformation. +// +// Arguments: +// pTempCache - pointer to the temp locals cache +// block - BasicBlock where the statement lives +// stmt - statement to walk +// +// Returns: +// True if anything changed, false if the IR was unchanged. +// +bool Compiler::fgMorphArrayOpsStmt(MorphMDArrayTempCache* pTempCache, BasicBlock* block, Statement* stmt) +{ + class MorphMDArrayVisitor final : public GenTreeVisitor + { + public: + enum + { + DoPostOrder = true + }; + + MorphMDArrayVisitor(Compiler* compiler, BasicBlock* block, MorphMDArrayTempCache* pTempCache) + : GenTreeVisitor(compiler), m_changed(false), m_block(block), m_pTempCache(pTempCache) + { + } + + bool Changed() const + { + return m_changed; + } + + fgWalkResult PostOrderVisit(GenTree** use, GenTree* user) + { + GenTree* const node = *use; + + if (!node->OperIs(GT_ARR_ELEM)) + { + return Compiler::WALK_CONTINUE; + } + + GenTreeArrElem* const arrElem = node->AsArrElem(); + + JITDUMP("Morphing GT_ARR_ELEM [%06u] in " FMT_BB " of '%s'\n", dspTreeID(arrElem), m_block->bbNum, + m_compiler->info.compFullName); + DISPTREE(arrElem); + + // impArrayAccessIntrinsic() ensures the following. + assert((2 <= arrElem->gtArrRank) && (arrElem->gtArrRank <= GT_ARR_MAX_RANK)); + assert(arrElem->gtArrObj->TypeIs(TYP_REF)); + assert(arrElem->TypeIs(TYP_BYREF)); + + for (unsigned i = 0; i < arrElem->gtArrRank; i++) + { + assert(arrElem->gtArrInds[i] != nullptr); + + // We cast the index operands to TYP_INT in the importer. + // Note that the offset calculcation needs to be TYP_I_IMPL, as multiplying the linearized index + // by the array element scale might overflow (although does .NET support array objects larger than + // 2GB in size?). + assert(genActualType(arrElem->gtArrInds[i]->TypeGet()) == TYP_INT); + } + + // The order of evaluation of a[i,j,k] is: a, i, j, k. That is, if any of the i, j, k throw an + // exception, it needs to happen before accessing `a`. For example, `a` could be null, but `i` + // could be an expresssion throwing an exception, and that exception needs to be thrown before + // indirecting using `a` (such as reading a dimension length or lower bound). + // + // First, we need to make temp copies of the index expressions that have side-effects. We + // always make a copy of the array object (below) so we can multi-use it. + // + GenTree* idxToUse[GT_ARR_MAX_RANK]; + unsigned idxToCopy[GT_ARR_MAX_RANK]; + bool anyIdxWithSideEffects = false; + for (unsigned i = 0; i < arrElem->gtArrRank; i++) + { + GenTree* idx = arrElem->gtArrInds[i]; + if ((idx->gtFlags & GTF_ALL_EFFECT) == 0) + { + // No side-effect; just use it. + idxToUse[i] = idx; + idxToCopy[i] = BAD_VAR_NUM; + } + else + { + // Side-effect; create a temp. + // unsigned newIdxLcl = m_compiler->lvaGrabTemp(true DEBUGARG("MD array index copy")); + unsigned newIdxLcl = m_pTempCache->GrabTemp(idx->TypeGet()); + GenTree* newIdx = m_compiler->gtNewLclvNode(newIdxLcl, idx->TypeGet()); + idxToUse[i] = newIdx; + idxToCopy[i] = newIdxLcl; + anyIdxWithSideEffects = true; + } + } + + // `newArrLcl` is set to the lclvar with a copy of the array object, if needed. The creation/copy of the + // array object to this lcl is done as a top-level comma if needed. + unsigned arrLcl = BAD_VAR_NUM; + unsigned newArrLcl = BAD_VAR_NUM; + GenTree* arrObj = arrElem->gtArrObj; + unsigned rank = arrElem->gtArrRank; + + // We are going to multiply reference the array object; create a new local var if necessary. + if (arrObj->OperIs(GT_LCL_VAR)) + { + arrLcl = arrObj->AsLclVar()->GetLclNum(); + } + else + { + // arrLcl = newArrLcl = m_compiler->lvaGrabTemp(true DEBUGARG("MD array copy")); + arrLcl = newArrLcl = m_pTempCache->GrabTemp(TYP_REF); + } + + GenTree* fullTree = nullptr; + + // Work from outer-to-inner rank (i.e., slowest-changing to fastest-changing index), building up the offset + // tree. + for (unsigned i = 0; i < arrElem->gtArrRank; i++) + { + GenTree* idx = idxToUse[i]; + assert((idx->gtFlags & GTF_ALL_EFFECT) == 0); // We should have taken care of side effects earlier. + + GenTreeMDArr* const mdArrLowerBound = + m_compiler->gtNewMDArrLowerBound(m_compiler->gtNewLclvNode(arrLcl, TYP_REF), i, rank, m_block); + // unsigned effIdxLcl = m_compiler->lvaGrabTemp(true DEBUGARG("MD array effective index")); + unsigned effIdxLcl = m_pTempCache->GrabTemp(TYP_INT); + GenTree* const effIndex = m_compiler->gtNewOperNode(GT_SUB, TYP_INT, idx, mdArrLowerBound); + GenTree* const asgNode = m_compiler->gtNewTempAssign(effIdxLcl, effIndex); + GenTreeMDArr* const mdArrLength = + m_compiler->gtNewMDArrLen(m_compiler->gtNewLclvNode(arrLcl, TYP_REF), i, rank, m_block); + GenTreeBoundsChk* const arrBndsChk = new (m_compiler, GT_BOUNDS_CHECK) + GenTreeBoundsChk(m_compiler->gtNewLclvNode(effIdxLcl, TYP_INT), mdArrLength, SCK_RNGCHK_FAIL); + GenTree* const boundsCheckComma = + m_compiler->gtNewOperNode(GT_COMMA, TYP_INT, arrBndsChk, + m_compiler->gtNewLclvNode(effIdxLcl, TYP_INT)); + GenTree* const idxComma = m_compiler->gtNewOperNode(GT_COMMA, TYP_INT, asgNode, boundsCheckComma); + + // If it's not the first index, accumulate with the previously created calculation. + if (i > 0) + { + assert(fullTree != nullptr); + + GenTreeMDArr* const mdArrLengthScale = + m_compiler->gtNewMDArrLen(m_compiler->gtNewLclvNode(arrLcl, TYP_REF), i, rank, m_block); + GenTree* const scale = m_compiler->gtNewOperNode(GT_MUL, TYP_INT, fullTree, mdArrLengthScale); + GenTree* const effIndex = m_compiler->gtNewOperNode(GT_ADD, TYP_INT, scale, idxComma); + + fullTree = effIndex; + } + else + { + fullTree = idxComma; + } + } + +#ifdef TARGET_64BIT + // Widen the linearized index on 64-bit targets; subsequent math will be done in TYP_I_IMPL. + assert(fullTree->TypeIs(TYP_INT)); + fullTree = m_compiler->gtNewCastNode(TYP_I_IMPL, fullTree, true, TYP_I_IMPL); +#endif // TARGET_64BIT + + // Now scale by element size and add offset from array object to array data base. + + unsigned elemScale = arrElem->gtArrElemSize; + unsigned dataOffset = m_compiler->eeGetMDArrayDataOffset(arrElem->gtArrRank); + GenTree* const scale = + m_compiler->gtNewOperNode(GT_MUL, TYP_I_IMPL, fullTree, + m_compiler->gtNewIconNode(static_cast(elemScale), TYP_I_IMPL)); + GenTree* const scalePlusOffset = + m_compiler->gtNewOperNode(GT_ADD, TYP_I_IMPL, scale, + m_compiler->gtNewIconNode(static_cast(dataOffset), TYP_I_IMPL)); + GenTree* fullExpansion = m_compiler->gtNewOperNode(GT_ADD, TYP_BYREF, scalePlusOffset, + m_compiler->gtNewLclvNode(arrLcl, TYP_REF)); + + // Add copies of the index expressions with side effects. Add them in reverse order, so the first index + // ends up at the top of the tree (so, first in execution order). + if (anyIdxWithSideEffects) + { + for (unsigned i = arrElem->gtArrRank; i > 0; i--) + { + if (idxToCopy[i - 1] != BAD_VAR_NUM) + { + GenTree* const idxLclAsg = + m_compiler->gtNewTempAssign(idxToCopy[i - 1], arrElem->gtArrInds[i - 1]); + fullExpansion = + m_compiler->gtNewOperNode(GT_COMMA, fullExpansion->TypeGet(), idxLclAsg, fullExpansion); + } + } + } + + // If we needed to create a new local for the array object, copy that before everything. + if (newArrLcl != BAD_VAR_NUM) + { + GenTree* const arrLclAsg = m_compiler->gtNewTempAssign(newArrLcl, arrObj); + fullExpansion = m_compiler->gtNewOperNode(GT_COMMA, fullExpansion->TypeGet(), arrLclAsg, fullExpansion); + } + + JITDUMP("fgMorphArrayOpsStmt (before remorph):\n"); + DISPTREE(fullExpansion); + + *use = fullExpansion; + m_changed = true; + + // The GT_ARR_ELEM node is no longer needed. + DEBUG_DESTROY_NODE(node); + + return fgWalkResult::WALK_CONTINUE; + } + + private: + bool m_changed; + BasicBlock* m_block; + MorphMDArrayTempCache* m_pTempCache; + }; + + MorphMDArrayVisitor morphMDArrayVisitor(this, block, pTempCache); + morphMDArrayVisitor.WalkTree(stmt->GetRootNodePointer(), nullptr); + return morphMDArrayVisitor.Changed(); +} + +//------------------------------------------------------------------------ +// fgMorphArrayOps: Morph multi-dimensional (MD) array operations in this method. +// +// GT_ARR_ELEM nodes are morphed to appropriate trees. Note that MD array `Get`, `Set`, or `Address` +// is imported as a call, and, if all required conditions are satisfied, is treated as an intrinsic +// and replaced by IR nodes, especially GT_ARR_ELEM nodes, in impArrayAccessIntrinsic(). +// +// For example, a simple 2-dimensional array access like `a[i,j]` looks like: +// +// \--* ARR_ELEM[,] byref +// +--* LCL_VAR ref V00 arg0 +// +--* LCL_VAR int V01 arg1 +// \--* LCL_VAR int V02 arg2 +// +// This is replaced by: +// +// &a + offset + elemSize * ((i - a.GetLowerBound(0)) * a.GetLength(1) + (j - a.GetLowerBound(1))) +// +// plus the appropriate `i` and `j` bounds checks. +// +// In IR, this is: +// +// * ADD byref +// +--* ADD long +// | +--* MUL long +// | | +--* CAST long <- uint +// | | | \--* ADD int +// | | | +--* MUL int +// | | | | +--* COMMA int +// | | | | | +--* ASG int +// | | | | | | +--* LCL_VAR int V04 tmp1 +// | | | | | | \--* SUB int +// | | | | | | +--* LCL_VAR int V01 arg1 +// | | | | | | \--* MDARR_LOWER_BOUND int (0) +// | | | | | | \--* LCL_VAR ref V00 arg0 +// | | | | | \--* COMMA int +// | | | | | +--* BOUNDS_CHECK_Rng void +// | | | | | | +--* LCL_VAR int V04 tmp1 +// | | | | | | \--* MDARR_LENGTH int (0) +// | | | | | | \--* LCL_VAR ref V00 arg0 +// | | | | | \--* LCL_VAR int V04 tmp1 +// | | | | \--* MDARR_LENGTH int (1) +// | | | | \--* LCL_VAR ref V00 arg0 +// | | | \--* COMMA int +// | | | +--* ASG int +// | | | | +--* LCL_VAR int V05 tmp2 +// | | | | \--* SUB int +// | | | | +--* LCL_VAR int V02 arg2 +// | | | | \--* MDARR_LOWER_BOUND int (1) +// | | | | \--* LCL_VAR ref V00 arg0 +// | | | \--* COMMA int +// | | | +--* BOUNDS_CHECK_Rng void +// | | | | +--* LCL_VAR int V05 tmp2 +// | | | | \--* MDARR_LENGTH int (1) +// | | | | \--* LCL_VAR ref V00 arg0 +// | | | \--* LCL_VAR int V05 tmp2 +// | | \--* CNS_INT long 4 +// | \--* CNS_INT long 32 +// \--* LCL_VAR ref V00 arg0 +// +// before being morphed by the usual morph transformations. +// +// Some things to consider: +// 1. MD have both a lower bound and length for each dimension (even if very few MD arrays actually have a +// lower bound) +// 2. GT_MDARR_LOWER_BOUND(dim) represents the lower-bound value for a particular array dimension. The "effective +// index" for a dimension is the index minus the lower bound. +// 3. GT_MDARR_LENGTH(dim) represents the length value (number of elements in a dimension) for a particular +// array dimension. +// 4. The effective index is bounds checked against the dimension length. +// 5. The lower bound and length values are 32-bit signed integers (TYP_INT). +// 6. After constructing a "linearized index", the index is scaled by the array element size, and the offset from +// the array object to the beginning of the array data is added. +// 7. Much of the complexity above is simply to assign temps to the various values that are used subsequently. +// 8. The index expressions are used exactly once. However, if have side effects, they need to be copied, early, +// to preserve exception ordering. +// 9. Only the top-level operation adds the array object to the scaled, linearized index, to create the final +// address `byref`. As usual, we need to be careful to not create an illegal byref by adding any partial index. +// calculation. +// 10. To avoid doing unnecessary work, the importer sets the global `OMF_HAS_MDARRAYREF` flag if there are any +// MD array expressions to expand. Also, the block flag `BBF_HAS_MDARRAYREF` is set to blocks where these exist, +// so only those blocks are processed. +// +// Returns: +// suitable phase status +// +PhaseStatus Compiler::fgMorphArrayOps() +{ + if (!opts.compJitEarlyExpandMDArrays) + { + JITDUMP("Early expansion of MD arrays disabled\n"); + return PhaseStatus::MODIFIED_NOTHING; + } + + if ((optMethodFlags & OMF_HAS_MDARRAYREF) == 0) + { + JITDUMP("No multi-dimensional array references in the function\n"); + return PhaseStatus::MODIFIED_NOTHING; + } + + // Maintain a cache of temp locals to use when we need a temp for this transformation. After each statement, + // reset the cache, meaning we can re-use any of the temps previously allocated. The idea here is to avoid + // creating too many temporaries, since the JIT has a limit on the number of tracked locals. A temp created + // here in one statement will have a distinct lifetime from a temp created in another statement, so register + // allocation is not constrained. + + bool changed = false; + MorphMDArrayTempCache mdArrayTempCache(this); + + for (BasicBlock* const block : Blocks()) + { + if ((block->bbFlags & BBF_HAS_MDARRAYREF) == 0) + { + // No MD array references in this block + continue; + } + + // Publish current block (needed for various morphing functions). + compCurBB = block; + + for (Statement* const stmt : block->Statements()) + { + if (fgMorphArrayOpsStmt(&mdArrayTempCache, block, stmt)) + { + changed = true; + + // Morph the statement if there have been changes. + + GenTree* tree = stmt->GetRootNode(); + GenTree* morphedTree = fgMorphTree(tree); + + JITDUMP("fgMorphArrayOps (after remorph):\n"); + DISPTREE(morphedTree); + + stmt->SetRootNode(morphedTree); + } + } + + mdArrayTempCache.Reset(); + } + + return changed ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING; +} diff --git a/src/coreclr/jit/optcse.cpp b/src/coreclr/jit/optcse.cpp index 05b14ac517610..d2cfa5b933f60 100644 --- a/src/coreclr/jit/optcse.cpp +++ b/src/coreclr/jit/optcse.cpp @@ -840,7 +840,7 @@ bool Compiler::optValnumCSE_Locate() noway_assert(((unsigned)tree->gtCSEnum) == CSEindex); } - if (IS_CSE_INDEX(CSEindex) && (tree->OperGet() == GT_ARR_LENGTH)) + if (IS_CSE_INDEX(CSEindex) && tree->OperIsArrLength()) { stmtHasArrLenCandidate = true; } @@ -3580,6 +3580,8 @@ bool Compiler::optIsCSEcandidate(GenTree* tree) case GT_ARR_ELEM: case GT_ARR_LENGTH: + case GT_MDARR_LENGTH: + case GT_MDARR_LOWER_BOUND: return true; case GT_LCL_VAR: diff --git a/src/coreclr/jit/optimizer.cpp b/src/coreclr/jit/optimizer.cpp index 2bd24b1223956..20504602a37d9 100644 --- a/src/coreclr/jit/optimizer.cpp +++ b/src/coreclr/jit/optimizer.cpp @@ -4671,7 +4671,7 @@ Compiler::fgWalkResult Compiler::optInvertCountTreeInfo(GenTree** pTree, fgWalkD o->sharedStaticHelperCount += 1; } - if ((*pTree)->OperGet() == GT_ARR_LENGTH) + if ((*pTree)->OperIsArrLength()) { o->arrayLengthCount += 1; } @@ -5001,9 +5001,9 @@ bool Compiler::optInvertWhileLoop(BasicBlock* block) assert(foundCondTree); // Flag the block that received the copy as potentially having an array/vtable - // reference, nullcheck, object/array allocation if the block copied from did; + // reference, nullcheck, object allocation if the block copied from did; // this is a conservative guess. - if (auto copyFlags = bTest->bbFlags & (BBF_HAS_IDX_LEN | BBF_HAS_NULLCHECK | BBF_HAS_NEWOBJ | BBF_HAS_NEWARRAY)) + if (auto copyFlags = bTest->bbFlags & (BBF_HAS_IDX_LEN | BBF_HAS_MD_IDX_LEN | BBF_HAS_NULLCHECK | BBF_HAS_NEWOBJ)) { bNewCond->bbFlags |= copyFlags; } @@ -6318,7 +6318,7 @@ void Compiler::optPerformHoistExpr(GenTree* origExpr, BasicBlock* exprBb, unsign compCurBB = preHead; hoist = fgMorphTree(hoist); - preHead->bbFlags |= (exprBb->bbFlags & (BBF_HAS_IDX_LEN | BBF_HAS_NULLCHECK)); + preHead->bbFlags |= (exprBb->bbFlags & (BBF_HAS_IDX_LEN | BBF_HAS_MD_IDX_LEN | BBF_HAS_NULLCHECK)); Statement* hoistStmt = gtNewStmt(hoist); diff --git a/src/coreclr/jit/valuenum.cpp b/src/coreclr/jit/valuenum.cpp index 4a6aeb158117a..a4549964c22e5 100644 --- a/src/coreclr/jit/valuenum.cpp +++ b/src/coreclr/jit/valuenum.cpp @@ -2894,8 +2894,11 @@ ValueNum ValueNumStore::EvalFuncForConstantArgs(var_types typ, VNFunc func, Valu // If arg0 has a possible exception, it wouldn't have been constant. assert(!VNHasExc(arg0VN)); // Otherwise... - assert(arg0VN == VNForNull()); // Only other REF constant. - assert(func == VNFunc(GT_ARR_LENGTH)); // Only function we can apply to a REF constant! + assert(arg0VN == VNForNull()); // Only other REF constant. + + // Only functions we can apply to a REF constant! + assert((func == VNFunc(GT_ARR_LENGTH)) || (func == VNF_MDArrLength) || + (func == VNFunc(GT_MDARR_LOWER_BOUND))); return VNWithExc(VNForVoid(), VNExcSetSingleton(VNForFunc(TYP_REF, VNF_NullPtrExc, VNForNull()))); } default: @@ -5691,13 +5694,15 @@ ValueNum ValueNumStore::GetArrForLenVn(ValueNum vn) } VNFuncApp funcAttr; - if (GetVNFunc(vn, &funcAttr) && funcAttr.m_func == (VNFunc)GT_ARR_LENGTH) + if (GetVNFunc(vn, &funcAttr) && + ((funcAttr.m_func == (VNFunc)GT_ARR_LENGTH) || (funcAttr.m_func == VNF_MDArrLength))) { return funcAttr.m_args[0]; } return NoVN; } +// TODO-MDArray: support JitNewMdArr, probably with a IsVNNewMDArr() function bool ValueNumStore::IsVNNewArr(ValueNum vn, VNFuncApp* funcApp) { if (vn == NoVN) @@ -5712,6 +5717,8 @@ bool ValueNumStore::IsVNNewArr(ValueNum vn, VNFuncApp* funcApp) return result; } +// TODO-MDArray: support array dimension length of a specific dimension for JitNewMdArr, with a GetNewMDArrSize() +// function. int ValueNumStore::GetNewArrSize(ValueNum vn) { VNFuncApp funcApp; @@ -5733,7 +5740,8 @@ bool ValueNumStore::IsVNArrLen(ValueNum vn) return false; } VNFuncApp funcAttr; - return (GetVNFunc(vn, &funcAttr) && funcAttr.m_func == (VNFunc)GT_ARR_LENGTH); + return GetVNFunc(vn, &funcAttr) && + ((funcAttr.m_func == (VNFunc)GT_ARR_LENGTH) || (funcAttr.m_func == VNF_MDArrLength)); } bool ValueNumStore::IsVNCheckedBound(ValueNum vn) @@ -6798,6 +6806,8 @@ static genTreeOps genTreeOpsIllegalAsVNFunc[] = {GT_IND, // When we do heap memo GT_OBJ, // May reference heap memory. GT_BLK, // May reference heap memory. GT_INIT_VAL, // Not strictly a pass-through. + GT_MDARR_LENGTH, + GT_MDARR_LOWER_BOUND, // 'dim' value must be considered // These control-flow operations need no values. GT_JTRUE, GT_RETURN, GT_SWITCH, GT_RETFILT, GT_CKFINITE}; @@ -8765,8 +8775,7 @@ void Compiler::fgValueNumberTree(GenTree* tree) // If we are fetching the array length for an array ref that came from global memory // then for CSE safety we must use the conservative value number for both // - if ((tree->OperGet() == GT_ARR_LENGTH) && - ((tree->AsOp()->gtOp1->gtFlags & GTF_GLOB_REF) != 0)) + if (tree->OperIsArrLength() && ((tree->AsOp()->gtOp1->gtFlags & GTF_GLOB_REF) != 0)) { // use the conservative value number for both when computing the VN for the ARR_LENGTH op1VNP.SetBoth(op1VNP.GetConservative()); @@ -8845,6 +8854,34 @@ void Compiler::fgValueNumberTree(GenTree* tree) fgValueNumberArrIndexAddr(tree->AsArrAddr()); break; + case GT_MDARR_LENGTH: + case GT_MDARR_LOWER_BOUND: + { + VNFunc mdarrVnf = (oper == GT_MDARR_LENGTH) ? VNF_MDArrLength : VNF_MDArrLowerBound; + GenTree* arrRef = tree->AsMDArr()->ArrRef(); + unsigned dim = tree->AsMDArr()->Dim(); + + ValueNumPair arrVNP; + ValueNumPair arrVNPx; + vnStore->VNPUnpackExc(arrRef->gtVNPair, &arrVNP, &arrVNPx); + + // If we are fetching the array length for an array ref that came from global memory + // then for CSE safety we must use the conservative value number for both. + // + if ((arrRef->gtFlags & GTF_GLOB_REF) != 0) + { + arrVNP.SetBoth(arrVNP.GetConservative()); + } + + ValueNumPair intPair; + intPair.SetBoth(vnStore->VNForIntCon(dim)); + + ValueNumPair normalPair = vnStore->VNPairForFunc(tree->TypeGet(), mdarrVnf, arrVNP, intPair); + + tree->gtVNPair = vnStore->VNPWithExc(normalPair, arrVNPx); + break; + } + case GT_BOUNDS_CHECK: { ValueNumPair vnpIndex = tree->AsBoundsChk()->GetIndex()->gtVNPair; @@ -9616,6 +9653,7 @@ void Compiler::fgValueNumberHelperCallFunc(GenTreeCall* call, VNFunc vnf, ValueN case VNF_JitNewMdArr: { + // TODO-MDArray: support value numbering new MD array helper generateUniqueVN = true; } break; @@ -10752,10 +10790,13 @@ void Compiler::fgValueNumberAddExceptionSet(GenTree* tree) break; case GT_ARR_LENGTH: - fgValueNumberAddExceptionSetForIndirection(tree, tree->AsArrLen()->ArrRef()); + case GT_MDARR_LENGTH: + case GT_MDARR_LOWER_BOUND: + fgValueNumberAddExceptionSetForIndirection(tree, tree->AsArrCommon()->ArrRef()); break; case GT_ARR_ELEM: + assert(!opts.compJitEarlyExpandMDArrays); fgValueNumberAddExceptionSetForIndirection(tree, tree->AsArrElem()->gtArrObj); break; diff --git a/src/coreclr/jit/valuenum.h b/src/coreclr/jit/valuenum.h index 5caf932e53217..e254bb4bf61e6 100644 --- a/src/coreclr/jit/valuenum.h +++ b/src/coreclr/jit/valuenum.h @@ -848,10 +848,10 @@ class ValueNumStore // Check if "vn" IsVNNewArr and return <= 0 if arr size cannot be determined, else array size. int GetNewArrSize(ValueNum vn); - // Check if "vn" is "a.len" + // Check if "vn" is "a.Length" or "a.GetLength(n)" bool IsVNArrLen(ValueNum vn); - // If "vn" is VN(a.len) then return VN(a); NoVN if VN(a) can't be determined. + // If "vn" is VN(a.Length) or VN(a.GetLength(n)) then return VN(a); NoVN if VN(a) can't be determined. ValueNum GetArrForLenVn(ValueNum vn); // Return true with any Relop except for == and != and one operand has to be a 32-bit integer constant. diff --git a/src/coreclr/jit/valuenumfuncs.h b/src/coreclr/jit/valuenumfuncs.h index 2bc7332afa4f1..31dbea82ff83d 100644 --- a/src/coreclr/jit/valuenumfuncs.h +++ b/src/coreclr/jit/valuenumfuncs.h @@ -23,6 +23,9 @@ ValueNumFuncDef(PtrToStatic, 3, false, true, false) // Pointer (byref) t // 1: (VN of) the field sequence, of which the first element is the static itself. // 2: (VN of) offset for the constituent struct fields +ValueNumFuncDef(MDArrLength, 2, false, false, false) // MD array len, Args: 0: array, 1: dimension +ValueNumFuncDef(MDArrLowerBound, 2, false, false, false) // MD array lower bound, Args: 0: array, 1: dimension + ValueNumFuncDef(InitVal, 1, false, false, false) // An input arg, or init val of a local Args: 0: a constant VN. ValueNumFuncDef(Cast, 2, false, false, false) // VNF_Cast: Cast Operation changes the representations size and unsigned-ness.