From 205ef031e0fe5152dede0bd9f99d0f6f9e7f1e45 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 22 Jan 2024 10:31:44 +0100 Subject: [PATCH] JIT: Remove old loop code (#97232) All dependencies on old loop finding have been removed, so remove all old code associated with it. --- src/coreclr/jit/block.cpp | 10 +- src/coreclr/jit/block.h | 12 +- src/coreclr/jit/compiler.cpp | 127 +- src/coreclr/jit/compiler.h | 388 +--- src/coreclr/jit/compiler.hpp | 202 -- src/coreclr/jit/fgbasic.cpp | 117 - src/coreclr/jit/fgdiagnostic.cpp | 510 +--- src/coreclr/jit/fgopt.cpp | 78 - src/coreclr/jit/flowgraph.cpp | 45 +- src/coreclr/jit/gentree.h | 1 - src/coreclr/jit/helperexpansion.cpp | 45 +- src/coreclr/jit/jitconfigvalues.h | 8 +- src/coreclr/jit/loopcloning.cpp | 10 +- src/coreclr/jit/loopcloning.h | 17 +- src/coreclr/jit/morph.cpp | 64 +- src/coreclr/jit/optimizebools.cpp | 12 +- src/coreclr/jit/optimizer.cpp | 2833 +++-------------------- src/coreclr/jit/phase.cpp | 2 +- src/coreclr/jit/redundantbranchopts.cpp | 36 - 19 files changed, 363 insertions(+), 4154 deletions(-) diff --git a/src/coreclr/jit/block.cpp b/src/coreclr/jit/block.cpp index 24d37e691c9ff..01e137432ecf4 100644 --- a/src/coreclr/jit/block.cpp +++ b/src/coreclr/jit/block.cpp @@ -491,7 +491,7 @@ void BasicBlock::dspFlags() const {BBF_INTERNAL, "internal"}, {BBF_FAILED_VERIFICATION, "failV"}, {BBF_HAS_SUPPRESSGC_CALL, "sup-gc"}, - {BBF_LOOP_HEAD, "Loop"}, + {BBF_LOOP_HEAD, "loophead"}, {BBF_HAS_LABEL, "label"}, {BBF_HAS_JMP, "jmp"}, {BBF_HAS_CALL, "hascall"}, @@ -513,7 +513,7 @@ void BasicBlock::dspFlags() const {BBF_NO_CSE_IN, "no-cse"}, {BBF_CAN_ADD_PRED, "add-pred"}, {BBF_RETLESS_CALL, "retless"}, - {BBF_LOOP_PREHEADER, "LoopPH"}, + {BBF_LOOP_PREHEADER, "preheader"}, {BBF_COLD, "cold"}, {BBF_KEEP_BBJ_ALWAYS, "KEEP"}, {BBF_CLONED_FINALLY_BEGIN, "cfb"}, @@ -811,7 +811,6 @@ void BasicBlock::CloneBlockState( to->bbCodeOffs = from->bbCodeOffs; to->bbCodeOffsEnd = from->bbCodeOffsEnd; VarSetOps::AssignAllowUninitRhs(compiler, to->bbScope, from->bbScope); - to->bbNatLoopNum = from->bbNatLoopNum; #ifdef DEBUG to->bbTgtStkDepth = from->bbTgtStkDepth; #endif // DEBUG @@ -1606,11 +1605,6 @@ BasicBlock* BasicBlock::New(Compiler* compiler) block->bbMemorySsaNumOut[memoryKind] = 0; } - // Make sure we reserve a NOT_IN_LOOP value that isn't a legal table index. - static_assert_no_msg(BasicBlock::MAX_LOOP_NUM < BasicBlock::NOT_IN_LOOP); - - block->bbNatLoopNum = BasicBlock::NOT_IN_LOOP; - block->bbPreorderNum = 0; block->bbPostorderNum = 0; diff --git a/src/coreclr/jit/block.h b/src/coreclr/jit/block.h index 2eef62974eed3..0760baaab0165 100644 --- a/src/coreclr/jit/block.h +++ b/src/coreclr/jit/block.h @@ -43,7 +43,7 @@ typedef BitVec_ValRet_T ASSERT_VALRET_TP; // This define is used with string concatenation to put this in printf format strings (Note that %u means unsigned int) #define FMT_BB "BB%02u" -// Use this format for loop table indices. +// Use this format for loop indices #define FMT_LP "L%02u" // And this format for profile weights @@ -393,7 +393,7 @@ enum BasicBlockFlags : unsigned __int64 BBF_HAS_NULLCHECK = MAKE_BBFLAG(11), // BB contains a null check BBF_HAS_SUPPRESSGC_CALL = MAKE_BBFLAG(12), // BB contains a call to a method with SuppressGCTransitionAttribute BBF_RUN_RARELY = MAKE_BBFLAG(13), // BB is rarely run (catch clauses, blocks with throws etc) - BBF_LOOP_HEAD = MAKE_BBFLAG(14), // BB is the head of a loop + BBF_LOOP_HEAD = MAKE_BBFLAG(14), // BB is the head of a loop (can reach a predecessor) BBF_HAS_LABEL = MAKE_BBFLAG(15), // BB needs a label BBF_LOOP_ALIGN = MAKE_BBFLAG(16), // Block is lexically the first block in a loop we intend to align. BBF_HAS_ALIGN = MAKE_BBFLAG(17), // BB ends with 'align' instruction @@ -1251,14 +1251,6 @@ struct BasicBlock : private LIR::Range #define BBCT_FILTER_HANDLER 0xFFFFFFFF #define handlerGetsXcptnObj(hndTyp) ((hndTyp) != BBCT_NONE && (hndTyp) != BBCT_FAULT && (hndTyp) != BBCT_FINALLY) - // The following fields are used for loop detection - typedef unsigned char loopNumber; - static const unsigned NOT_IN_LOOP = UCHAR_MAX; - static const unsigned MAX_LOOP_NUM = 64; - - loopNumber bbNatLoopNum; // Index, in optLoopTable, of most-nested loop that contains this block, - // or else NOT_IN_LOOP if this block is not in a loop. - // TODO-Cleanup: Get rid of bbStkDepth and use bbStackDepthOnEntry() instead union { unsigned short bbStkDepth; // stack depth on entry diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 9e88c6ab09130..7fed233ed8cce 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -305,7 +305,6 @@ Histogram computeReachabilitySetsIterationTable(computeReachabilitySetsIteration unsigned totalLoopMethods; // counts the total number of methods that have natural loops unsigned maxLoopsPerMethod; // counts the maximum number of loops a method has -unsigned totalLoopOverflows; // # of methods that identified more loops than we can represent unsigned totalLoopCount; // counts the total number of natural loops unsigned totalUnnatLoopCount; // counts the total number of (not-necessarily natural) loops unsigned totalUnnatLoopOverflows; // # of methods that identified more unnatural loops than we can represent @@ -1576,7 +1575,6 @@ void Compiler::compShutdown() jitprintf("Total number of methods with loops is %5u\n", totalLoopMethods); jitprintf("Total number of loops is %5u\n", totalLoopCount); jitprintf("Maximum number of loops per method is %5u\n", maxLoopsPerMethod); - jitprintf("# of methods overflowing nat loop table is %5u\n", totalLoopOverflows); jitprintf("Total number of 'unnatural' loops is %5u\n", totalUnnatLoopCount); jitprintf("# of methods overflowing unnat loop limit is %5u\n", totalUnnatLoopOverflows); jitprintf("Total number of loops with an iterator is %5u\n", iterLoopCount); @@ -5254,10 +5252,9 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl // // Remarks: // All innermost loops whose block weight meets a threshold are candidates for alignment. -// The top block of the loop is marked with the BBF_LOOP_ALIGN flag to indicate this -// (the loop table itself is not changed). +// The top block of the loop is marked with the BBF_LOOP_ALIGN flag to indicate this. // -// Depends on the loop table, and on block weights being set. +// Depends on block weights being set. // bool Compiler::shouldAlignLoop(FlowGraphNaturalLoop* loop, BasicBlock* top) { @@ -5839,11 +5836,10 @@ void Compiler::RecomputeFlowGraphAnnotations() fgComputeReachability(); optSetBlockWeights(); - optFindLoops(); fgInvalidateDfsTree(); m_dfsTree = fgComputeDfs(); - optFindNewLoops(); + optFindLoops(); if (fgHasLoops) { @@ -5851,13 +5847,6 @@ void Compiler::RecomputeFlowGraphAnnotations() } m_domTree = FlowGraphDominatorTree::Build(m_dfsTree); - - // Dominators and the loop table are computed above for old<->new loop - // crossreferencing, but they are not actually used for optimization - // anymore. - optLoopTableValid = false; - optLoopTable = nullptr; - optLoopCount = 0; } /*****************************************************************************/ @@ -9084,7 +9073,7 @@ void JitTimer::PrintCsvMethodStats(Compiler* comp) fprintf(s_csvFile, "%u,", comp->info.compILCodeSize); fprintf(s_csvFile, "%u,", comp->fgBBcount); fprintf(s_csvFile, "%u,", comp->opts.MinOpts()); - fprintf(s_csvFile, "%u,", comp->optLoopCount); + fprintf(s_csvFile, "%u,", comp->optNumNaturalLoopsFound); fprintf(s_csvFile, "%u,", comp->optLoopsCloned); #if FEATURE_LOOP_ALIGN #ifdef DEBUG @@ -9337,16 +9326,11 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX * cCVarSet, dCVarSet : Display a "converted" VARSET_TP: the varset is assumed to be tracked variable * indices. These are converted to variable numbers and sorted. (Calls * dumpConvertedVarSet()). - * cLoop, dLoop : Display the blocks of a loop, including the trees, given a loop number (call - * optPrintLoopInfo()). - * cLoopPtr, dLoopPtr : Display the blocks of a loop, including the trees, given a LoopDsc* (call - * optPrintLoopInfo()). - * cLoops, dLoops : Display the loop table (call optPrintLoopTable()). - * cNewLoops, dNewLoops : Display the loop table (call FlowGraphNaturalLoops::Dump()) with + * cLoops, dLoops : Display the loops (call FlowGraphNaturalLoops::Dump()) with * Compiler::m_loops. - * cNewLoopsA, dNewLoopsA : Display the loop table (call FlowGraphNaturalLoops::Dump()) with a given + * cLoopsA, dLoopsA : Display the loops (call FlowGraphNaturalLoops::Dump()) with a given * loops arg. - * cNewLoop, dNewLoop : Display a single loop (call FlowGraphNaturalLoop::Dump()) with given + * cLoop, dLoop : Display a single loop (call FlowGraphNaturalLoop::Dump()) with given * loop arg. * cTreeFlags, dTreeFlags : Display tree flags for a specified tree. * cVN, dVN : Display a ValueNum (call vnPrint()). @@ -9361,7 +9345,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX * dFindTree : Find a tree in the IR, specifying a tree id. Sets `dbTree` and `dbTreeBlock`. * dFindStmt : Find a Statement in the IR, specifying a statement id. Sets `dbStmt`. * dFindBlock : Find a block in the IR, specifying a block number. Sets `dbBlock`. - * dFindLoop : Find a loop in the loop table, specifying a loop number. Sets `dbLoop`. + * dFindLoop : Find a loop specifying a loop index. Sets `dbLoop`. */ // Make the debug helpers available (under #ifdef DEBUG) even though they are unreferenced. When the Microsoft @@ -9401,12 +9385,9 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #pragma comment(linker, "/include:cDoms") #pragma comment(linker, "/include:cLiveness") #pragma comment(linker, "/include:cCVarSet") -#pragma comment(linker, "/include:cLoop") -#pragma comment(linker, "/include:cLoopPtr") #pragma comment(linker, "/include:cLoops") -#pragma comment(linker, "/include:cNewLoops") -#pragma comment(linker, "/include:cNewLoopsA") -#pragma comment(linker, "/include:cNewLoop") +#pragma comment(linker, "/include:cLoopsA") +#pragma comment(linker, "/include:cLoop") #pragma comment(linker, "/include:cTreeFlags") #pragma comment(linker, "/include:cVN") @@ -9431,11 +9412,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #pragma comment(linker, "/include:dLiveness") #pragma comment(linker, "/include:dCVarSet") #pragma comment(linker, "/include:dLoop") -#pragma comment(linker, "/include:dLoopPtr") #pragma comment(linker, "/include:dLoops") -#pragma comment(linker, "/include:dNewLoops") -#pragma comment(linker, "/include:dNewLoopsA") -#pragma comment(linker, "/include:dNewLoop") #pragma comment(linker, "/include:dTreeFlags") #pragma comment(linker, "/include:dVN") @@ -9473,11 +9450,8 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #pragma comment(linker, "/include:_cLiveness") #pragma comment(linker, "/include:_cCVarSet") #pragma comment(linker, "/include:_cLoop") -#pragma comment(linker, "/include:_cLoopPtr") #pragma comment(linker, "/include:_cLoops") -#pragma comment(linker, "/include:_cNewLoops") -#pragma comment(linker, "/include:_cNewLoopsA") -#pragma comment(linker, "/include:_cNewLoop") +#pragma comment(linker, "/include:_cLoopsA") #pragma comment(linker, "/include:_cTreeFlags") #pragma comment(linker, "/include:_cVN") @@ -9501,12 +9475,9 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #pragma comment(linker, "/include:_dDoms") #pragma comment(linker, "/include:_dLiveness") #pragma comment(linker, "/include:_dCVarSet") -#pragma comment(linker, "/include:_dLoop") -#pragma comment(linker, "/include:_dLoopPtr") #pragma comment(linker, "/include:_dLoops") -#pragma comment(linker, "/include:_dNewLoops") -#pragma comment(linker, "/include:_dNewLoopsA") -#pragma comment(linker, "/include:_dNewLoop") +#pragma comment(linker, "/include:_dLoopsA") +#pragma comment(linker, "/include:_dLoop") #pragma comment(linker, "/include:_dTreeFlags") #pragma comment(linker, "/include:_dVN") @@ -9682,44 +9653,21 @@ JITDBGAPI void __cdecl cCVarSet(Compiler* comp, VARSET_VALARG_TP vars) printf("\n"); // dumpConvertedVarSet() doesn't emit a trailing newline } -JITDBGAPI void __cdecl cLoop(Compiler* comp, unsigned loopNum) -{ - static unsigned sequenceNumber = 0; // separate calls with a number to indicate this function has been called - printf("===================================================================== *Loop %u\n", sequenceNumber++); - comp->optPrintLoopInfo(loopNum, /* verbose */ true); - printf("\n"); -} - -JITDBGAPI void __cdecl cLoopPtr(Compiler* comp, const Compiler::LoopDsc* loop) -{ - static unsigned sequenceNumber = 0; // separate calls with a number to indicate this function has been called - printf("===================================================================== *LoopPtr %u\n", sequenceNumber++); - comp->optPrintLoopInfo(loop, /* verbose */ true); - printf("\n"); -} - JITDBGAPI void __cdecl cLoops(Compiler* comp) -{ - static unsigned sequenceNumber = 0; // separate calls with a number to indicate this function has been called - printf("===================================================================== *Loops %u\n", sequenceNumber++); - comp->optPrintLoopTable(); -} - -JITDBGAPI void __cdecl cNewLoops(Compiler* comp) { static unsigned sequenceNumber = 0; // separate calls with a number to indicate this function has been called printf("===================================================================== *NewLoops %u\n", sequenceNumber++); FlowGraphNaturalLoops::Dump(comp->m_loops); } -JITDBGAPI void __cdecl cNewLoopsA(Compiler* comp, FlowGraphNaturalLoops* loops) +JITDBGAPI void __cdecl cLoopsA(Compiler* comp, FlowGraphNaturalLoops* loops) { static unsigned sequenceNumber = 0; // separate calls with a number to indicate this function has been called printf("===================================================================== *NewLoopsA %u\n", sequenceNumber++); FlowGraphNaturalLoops::Dump(loops); } -JITDBGAPI void __cdecl cNewLoop(Compiler* comp, FlowGraphNaturalLoop* loop) +JITDBGAPI void __cdecl cLoop(Compiler* comp, FlowGraphNaturalLoop* loop) { static unsigned sequenceNumber = 0; // separate calls with a number to indicate this function has been called printf("===================================================================== *NewLoop %u\n", sequenceNumber++); @@ -9778,10 +9726,6 @@ JITDBGAPI void __cdecl cTreeFlags(Compiler* comp, GenTree* tree) { chars += printf("[VAR_USEASG]"); } - if (tree->gtFlags & GTF_VAR_ITERATOR) - { - chars += printf("[VAR_ITERATOR]"); - } if (tree->gtFlags & GTF_VAR_MOREUSES) { chars += printf("[VAR_MOREUSES]"); @@ -10318,34 +10262,19 @@ JITDBGAPI void __cdecl dCVarSet(VARSET_VALARG_TP vars) cCVarSet(JitTls::GetCompiler(), vars); } -JITDBGAPI void __cdecl dLoop(unsigned loopNum) -{ - cLoop(JitTls::GetCompiler(), loopNum); -} - -JITDBGAPI void __cdecl dLoopPtr(const Compiler::LoopDsc* loop) -{ - cLoopPtr(JitTls::GetCompiler(), loop); -} - JITDBGAPI void __cdecl dLoops() { cLoops(JitTls::GetCompiler()); } -JITDBGAPI void __cdecl dNewLoops() -{ - cNewLoops(JitTls::GetCompiler()); -} - -JITDBGAPI void __cdecl dNewLoopsA(FlowGraphNaturalLoops* loops) +JITDBGAPI void __cdecl dLoopsA(FlowGraphNaturalLoops* loops) { - cNewLoopsA(JitTls::GetCompiler(), loops); + cLoopsA(JitTls::GetCompiler(), loops); } -JITDBGAPI void __cdecl dNewLoop(FlowGraphNaturalLoop* loop) +JITDBGAPI void __cdecl dLoop(FlowGraphNaturalLoop* loop) { - cNewLoop(JitTls::GetCompiler(), loop); + cLoop(JitTls::GetCompiler(), loop); } JITDBGAPI void __cdecl dTreeFlags(GenTree* tree) @@ -10382,11 +10311,11 @@ JITDBGAPI void __cdecl dBlockList(BasicBlockList* list) // Trees, Stmts, and/or Blocks using id or bbNum. // That can be used in watch window or as a way to get address of fields for data breakpoints. -GenTree* dbTree; -Statement* dbStmt; -BasicBlock* dbTreeBlock; -BasicBlock* dbBlock; -Compiler::LoopDsc* dbLoop; +GenTree* dbTree; +Statement* dbStmt; +BasicBlock* dbTreeBlock; +BasicBlock* dbBlock; +FlowGraphNaturalLoop* dbLoop; // Debug APIs for finding Trees, Stmts, and/or Blocks. // As a side effect, they set the debug variables above. @@ -10478,18 +10407,18 @@ JITDBGAPI BasicBlock* __cdecl dFindBlock(unsigned bbNum) return nullptr; } -JITDBGAPI Compiler::LoopDsc* __cdecl dFindLoop(unsigned loopNum) +JITDBGAPI FlowGraphNaturalLoop* __cdecl dFindLoop(unsigned index) { Compiler* comp = JitTls::GetCompiler(); dbLoop = nullptr; - if (loopNum >= comp->optLoopCount) + if ((comp->m_loops == nullptr) || (index >= comp->m_loops->NumLoops())) { - printf("loopNum %u out of range\n", loopNum); + printf("Index %u out of range\n", index); return nullptr; } - dbLoop = &comp->optLoopTable[loopNum]; + dbLoop = comp->m_loops->GetLoopByIndex(index); return dbLoop; } diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 5bf68dfff8bbd..0f979e4700938 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -1516,7 +1516,7 @@ enum class PhaseChecks : unsigned int CHECK_UNIQUE = 1 << 1, // tree node uniqueness CHECK_FG = 1 << 2, // flow graph integrity CHECK_EH = 1 << 3, // eh table integrity - CHECK_LOOPS = 1 << 4, // loop table integrity + CHECK_LOOPS = 1 << 4, // loop integrity/canonicalization CHECK_PROFILE = 1 << 5, // profile data integrity CHECK_LINKED_LOCALS = 1 << 6, // check linked list of locals }; @@ -1887,60 +1887,6 @@ typedef JitHashTable, TestLabelAndNum> NodeToT // XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #endif // DEBUG -//------------------------------------------------------------------------- -// LoopFlags: flags for the loop table. -// -enum LoopFlags : unsigned short -{ - LPFLG_EMPTY = 0, - - // LPFLG_UNUSED = 0x0001, - // LPFLG_UNUSED = 0x0002, - LPFLG_ITER = 0x0004, // loop of form: for (i = icon or expression; test_condition(); i++) - // LPFLG_UNUSED = 0x0008, - - LPFLG_CONTAINS_CALL = 0x0010, // If executing the loop body *may* execute a call - // LPFLG_UNUSED = 0x0020, - LPFLG_CONST_INIT = 0x0040, // iterator is initialized with a constant (found in lpConstInit) - LPFLG_SIMD_LIMIT = 0x0080, // iterator is compared with vector element count (found in lpConstLimit) - - LPFLG_VAR_LIMIT = 0x0100, // iterator is compared with a local var (var # found in lpVarLimit) - LPFLG_CONST_LIMIT = 0x0200, // iterator is compared with a constant (found in lpConstLimit) - LPFLG_ARRLEN_LIMIT = 0x0400, // iterator is compared with a.len or a[i].len (found in lpArrLenLimit) - LPFLG_HAS_PREHEAD = 0x0800, // lpHead is known to be a preHead for this loop - - LPFLG_REMOVED = 0x1000, // has been removed from the loop table (unrolled or optimized away) - LPFLG_DONT_UNROLL = 0x2000, // do not unroll this loop - LPFLG_ASGVARS_YES = 0x4000, // "lpAsgVars" has been computed - LPFLG_ASGVARS_INC = 0x8000, // "lpAsgVars" is incomplete -- vars beyond those representable in an AllVarSet - // type are assigned to. -}; - -inline constexpr LoopFlags operator~(LoopFlags a) -{ - return (LoopFlags)(~(unsigned short)a); -} - -inline constexpr LoopFlags operator|(LoopFlags a, LoopFlags b) -{ - return (LoopFlags)((unsigned short)a | (unsigned short)b); -} - -inline constexpr LoopFlags operator&(LoopFlags a, LoopFlags b) -{ - return (LoopFlags)((unsigned short)a & (unsigned short)b); -} - -inline LoopFlags& operator|=(LoopFlags& a, LoopFlags b) -{ - return a = (LoopFlags)((unsigned short)a | (unsigned short)b); -} - -inline LoopFlags& operator&=(LoopFlags& a, LoopFlags b) -{ - return a = (LoopFlags)((unsigned short)a & (unsigned short)b); -} - // Represents a depth-first search tree of the flow graph. class FlowGraphDfsTree { @@ -5014,6 +4960,9 @@ class Compiler FlowGraphDominatorTree* m_domTree; BlockReachabilitySets* m_reachabilitySets; + bool optLoopsRequirePreHeaders; // Do we require that all loops (in m_loops) have pre-headers? + unsigned optNumNaturalLoopsFound; // Number of natural loops found in the loop finding phase + bool fgBBVarSetsInited; // Track how many artificial ref counts we've added to fgEntryBB (for OSR) @@ -5970,8 +5919,6 @@ class Compiler void fgCompactBlocks(BasicBlock* block, BasicBlock* bNext); - void fgUpdateLoopsAfterCompacting(BasicBlock* block, BasicBlock* bNext); - BasicBlock* fgConnectFallThrough(BasicBlock* bSrc, BasicBlock* bDst, bool noFallThroughQuirk = false); bool fgRenumberBlocks(); @@ -6107,7 +6054,7 @@ class Compiler void fgDebugCheckNodeLinks(BasicBlock* block, Statement* stmt); void fgDebugCheckLinkedLocals(); void fgDebugCheckNodesUniqueness(); - void fgDebugCheckLoopTable(); + void fgDebugCheckLoops(); void fgDebugCheckSsa(); void fgDebugCheckTypes(GenTree* tree); @@ -6798,13 +6745,6 @@ class Compiler // Compute the sets of long and float vars (lvaLongVars, lvaFloatVars). void optComputeInterestingVarSets(); -#ifdef DEBUG - bool optAnyChildNotRemoved(unsigned loopNum); -#endif // DEBUG - - // Mark a loop as removed. - void optMarkLoopRemoved(unsigned loopNum); - private: // Given a loop mark it and any nested loops as having 'memoryHavoc' void optRecordLoopNestsMemoryHavoc(FlowGraphNaturalLoop* loop, MemoryKindSet memoryHavoc); @@ -6828,7 +6768,6 @@ class Compiler PhaseStatus optFindLoopsPhase(); // Finds loops and records them in the loop table void optFindLoops(); - void optFindNewLoops(); bool optCanonicalizeLoops(); bool optCompactLoops(); bool optCompactLoop(FlowGraphNaturalLoop* loop); @@ -6845,268 +6784,13 @@ class Compiler void optRemoveRedundantZeroInits(); PhaseStatus optIfConversion(); // If conversion -protected: - // This enumeration describes what is killed by a call. - - enum callInterf - { - CALLINT_NONE, // no interference (most helpers) - CALLINT_REF_INDIRS, // kills GC ref indirections (SETFIELD OBJ) - CALLINT_SCL_INDIRS, // kills non GC ref indirections (SETFIELD non-OBJ) - CALLINT_ALL_INDIRS, // kills both GC ref and non GC ref indirections (SETFIELD STRUCT) - CALLINT_ALL, // kills everything (normal method call) - }; - -public: - // A "LoopDsc" describes a ("natural") loop. We (currently) require the body of a loop to be a contiguous (in - // bbNext order) sequence of basic blocks. (At times, we may require the blocks in a loop to be "properly numbered" - // in bbNext order; we use comparisons on the bbNum to decide order.) - // The blocks that define the body are - // top <= entry <= bottom - // The "head" of the loop is a block outside the loop that has "entry" as a successor. We only support loops with a - // single 'head' block. The meanings of these blocks are given in the definitions below. Also see the picture at - // Compiler::optFindNaturalLoops(). - struct LoopDsc - { - BasicBlock* lpHead; // HEAD of the loop (not part of the looping of the loop) -- has ENTRY as a successor. - BasicBlock* lpTop; // loop TOP (the back edge from lpBottom reaches here). Lexically first block (in bbNext - // order) reachable in this loop. - BasicBlock* lpEntry; // the ENTRY in the loop (in most cases TOP or BOTTOM) - BasicBlock* lpBottom; // loop BOTTOM (from here we have a back edge to the TOP) - BasicBlock* lpExit; // if a single exit loop this is the EXIT (in most cases BOTTOM) - - callInterf lpAsgCall; // "callInterf" for calls in the loop - ALLVARSET_TP lpAsgVars; // set of vars assigned within the loop (all vars, not just tracked) - varRefKinds lpAsgInds : 8; // set of inds modified within the loop - - LoopFlags lpFlags; - - unsigned char lpExitCnt; // number of exits from the loop - - unsigned char lpParent; // The index of the most-nested loop that completely contains this one, - // or else BasicBlock::NOT_IN_LOOP if no such loop exists. - unsigned char lpChild; // The index of a nested loop, or else BasicBlock::NOT_IN_LOOP if no child exists. - // (Actually, an "immediately" nested loop -- - // no other child of this loop is a parent of lpChild.) - unsigned char lpSibling; // The index of another loop that is an immediate child of lpParent, - // or else BasicBlock::NOT_IN_LOOP. One can enumerate all the children of a loop - // by following "lpChild" then "lpSibling" links. - - bool lpLoopHasMemoryHavoc[MemoryKindCount]; // The loop contains an operation that we assume has arbitrary - // memory side effects. If this is set, the fields below - // may not be accurate (since they become irrelevant.) - - VARSET_TP lpVarInOut; // The set of variables that are IN or OUT during the execution of this loop - VARSET_TP lpVarUseDef; // The set of variables that are USE or DEF during the execution of this loop - - // The following counts are used for hoisting profitability checks. - - int lpHoistedExprCount; // The register count for the non-FP expressions from inside this loop that have been - // hoisted - int lpLoopVarCount; // The register count for the non-FP LclVars that are read/written inside this loop - int lpVarInOutCount; // The register count for the non-FP LclVars that are alive inside or across this loop - - int lpHoistedFPExprCount; // The register count for the FP expressions from inside this loop that have been - // hoisted - int lpLoopVarFPCount; // The register count for the FP LclVars that are read/written inside this loop - int lpVarInOutFPCount; // The register count for the FP LclVars that are alive inside or across this loop - - typedef JitHashTable, FieldKindForVN> - FieldHandleSet; - FieldHandleSet* lpFieldsModified; // This has entries for all static field and object instance fields modified - // in the loop. - - typedef JitHashTable, bool> ClassHandleSet; - ClassHandleSet* lpArrayElemTypesModified; // Bits set indicate the set of sz array element types such that - // arrays of that type are modified - // in the loop. - - /* The following values are set only for iterator loops, i.e. has the flag LPFLG_ITER set */ - - GenTree* lpIterTree; // The "i = i const" tree - unsigned lpIterVar() const; // iterator variable # - int lpIterConst() const; // the constant with which the iterator is incremented - genTreeOps lpIterOper() const; // the type of the operation on the iterator (ADD, SUB, etc.) - void VERIFY_lpIterTree() const; - - var_types lpIterOperType() const; // For overflow instructions - - // Set to the block where we found the initialization for LPFLG_CONST_INIT loops. - // Initially, this will be 'head', but 'head' might change if we insert a loop pre-header block. - BasicBlock* lpInitBlock; - - int lpConstInit; // initial constant value of iterator : Valid if LPFLG_CONST_INIT - - // The following is for LPFLG_ITER loops only (i.e. the loop condition is "i RELOP const or var") - - GenTree* lpTestTree; // pointer to the node containing the loop test - genTreeOps lpTestOper() const; // the type of the comparison between the iterator and the limit (GT_LE, GT_GE, - // etc.) - - bool lpIsIncreasingLoop() const; // if the loop iterator increases from low to high value. - bool lpIsDecreasingLoop() const; // if the loop iterator decreases from high to low value. - - void VERIFY_lpTestTree() const; - - bool lpIsReversed() const; // true if the iterator node is the second operand in the loop condition - GenTree* lpIterator() const; // the iterator node in the loop test - GenTree* lpLimit() const; // the limit node in the loop test - - // Limit constant value of iterator - loop condition is "i RELOP const" - // : Valid if LPFLG_CONST_LIMIT - int lpConstLimit() const; - - // The lclVar # in the loop condition ( "i RELOP lclVar" ) - // : Valid if LPFLG_VAR_LIMIT - unsigned lpVarLimit() const; - - // The array length in the loop condition ( "i RELOP arr.len" or "i RELOP arr[i][j].len" ) - // : Valid if LPFLG_ARRLEN_LIMIT - bool lpArrLenLimit(Compiler* comp, ArrIndex* index) const; - - // Returns "true" iff this is a "top entry" loop. - bool lpIsTopEntry() const - { - if (lpHead->NextIs(lpEntry)) - { - assert(lpHead->bbFallsThrough() || lpHead->JumpsToNext()); - assert(lpTop == lpEntry); - return true; - } - else - { - return false; - } - } - - // Returns "true" iff this is removed loop. - bool lpIsRemoved() const - { - return (lpFlags & LPFLG_REMOVED) != 0; - } - - // Returns "true" iff "*this" contains the blk. - bool lpContains(BasicBlock* blk) const - { - return lpTop->bbNum <= blk->bbNum && blk->bbNum <= lpBottom->bbNum; - } - - // Returns "true" iff "*this" (properly) contains the range [top, bottom] (allowing tops - // to be equal, but requiring bottoms to be different.) - bool lpContains(BasicBlock* top, BasicBlock* bottom) const - { - return lpTop->bbNum <= top->bbNum && bottom->bbNum < lpBottom->bbNum; - } - - // Returns "true" iff "*this" (properly) contains "lp2" (allowing tops to be equal, but requiring - // bottoms to be different.) - bool lpContains(const LoopDsc& lp2) const - { - return lpContains(lp2.lpTop, lp2.lpBottom); - } - - // Returns "true" iff "*this" is (properly) contained by the range [top, bottom] - // (allowing tops to be equal, but requiring bottoms to be different.) - bool lpContainedBy(BasicBlock* top, BasicBlock* bottom) const - { - return top->bbNum <= lpTop->bbNum && lpBottom->bbNum < bottom->bbNum; - } - - // Returns "true" iff "*this" is (properly) contained by "lp2" - // (allowing tops to be equal, but requiring bottoms to be different.) - bool lpContainedBy(const LoopDsc& lp2) const - { - return lpContainedBy(lp2.lpTop, lp2.lpBottom); - } - - // Returns "true" iff "*this" is disjoint from the range [top, bottom]. - bool lpDisjoint(BasicBlock* top, BasicBlock* bottom) const - { - return bottom->bbNum < lpTop->bbNum || lpBottom->bbNum < top->bbNum; - } - // Returns "true" iff "*this" is disjoint from "lp2". - bool lpDisjoint(const LoopDsc& lp2) const - { - return lpDisjoint(lp2.lpTop, lp2.lpBottom); - } - - // Returns "true" iff the loop is well-formed (see code for defn). - bool lpWellFormed() const - { - return lpTop->bbNum <= lpEntry->bbNum && lpEntry->bbNum <= lpBottom->bbNum && - (lpHead->bbNum < lpTop->bbNum || lpHead->bbNum > lpBottom->bbNum); - } - -#ifdef DEBUG - void lpValidatePreHeader() const - { - // If this is called, we expect there to be a pre-header. - assert(lpFlags & LPFLG_HAS_PREHEAD); - - // The pre-header must unconditionally enter the loop. - assert(lpHead->GetUniqueSucc() == lpEntry); - - // The loop block must be marked as a pre-header. - assert(lpHead->HasFlag(BBF_LOOP_PREHEADER)); - - // The loop entry must have a single non-loop predecessor, which is the pre-header. - // We can't assume here that the bbNum are properly ordered, so we can't do a simple lpContained() - // check. So, we defer this check, which will be done by `fgDebugCheckLoopTable()`. - } -#endif // DEBUG - - // LoopBlocks: convenience method for enabling range-based `for` iteration over all the - // blocks in a loop, e.g.: - // for (BasicBlock* const block : loop->LoopBlocks()) ... - // Currently, the loop blocks are expected to be in linear, lexical, `bbNext` order - // from `lpTop` through `lpBottom`, inclusive. All blocks in this range are considered - // to be part of the loop. - // - BasicBlockRangeList LoopBlocks() const - { - return BasicBlockRangeList(lpTop, lpBottom); - } - }; - -protected: - bool fgMightHaveLoop(); // returns true if there are any back edges - bool fgHasLoops; // True if this method has any loops, set in fgComputeReachability - public: - LoopDsc* optLoopTable; // loop descriptor table - bool optLoopTableValid; // info in loop table should be valid - bool optLoopsRequirePreHeaders; // Do we require that all loops (in the loop table) have pre-headers? - unsigned char optLoopCount; // number of tracked loops - - // Every time we rebuild the loop table, we increase the global "loop epoch". Any loop indices or - // loop table pointers from the previous epoch are invalid. - // TODO: validate this in some way? - unsigned optCurLoopEpoch; - - void NewLoopEpoch() - { - ++optCurLoopEpoch; - JITDUMP("New loop epoch %d\n", optCurLoopEpoch); - } - + bool fgHasLoops; #ifdef DEBUG unsigned loopAlignCandidates; // number of candidates identified by placeLoopAlignInstructions unsigned loopsAligned; // number of loops actually aligned #endif // DEBUG - bool optRecordLoop(BasicBlock* head, - BasicBlock* top, - BasicBlock* entry, - BasicBlock* bottom, - BasicBlock* exit, - unsigned char exitCnt); - -#ifdef DEBUG - void optPrintLoopInfo(unsigned lnum, bool printVerbose = false); - void optPrintLoopInfo(const LoopDsc* loop, bool printVerbose = false); - void optPrintLoopTable(); -#endif - protected: unsigned optCallCount; // number of calls made in the method unsigned optIndirectCallCount; // number of virtual, interface and indirect calls made in the method @@ -7125,34 +6809,11 @@ class Compiler void optScaleLoopBlocks(BasicBlock* begBlk, BasicBlock* endBlk); - void optUnmarkLoopBlocks(BasicBlock* begBlk, BasicBlock* endBlk); - - void optUpdateLoopsBeforeRemoveBlock(BasicBlock* block, bool skipUnmarkLoop = false); - bool optIsLoopTestEvalIntoTemp(Statement* testStmt, Statement** newTestStmt); unsigned optIsLoopIncrTree(GenTree* incr); - bool optCheckIterInLoopTest(unsigned loopInd, GenTree* test, unsigned iterVar); - bool optComputeIterInfo(GenTree* incr, BasicBlock* from, BasicBlock* to, unsigned* pIterVar); - bool optPopulateInitInfo(unsigned loopInd, BasicBlock* initBlock, GenTree* init, unsigned iterVar); bool optExtractInitTestIncr( BasicBlock** pInitBlock, BasicBlock* bottom, BasicBlock* top, GenTree** ppInit, GenTree** ppTest, GenTree** ppIncr); - void optFindNaturalLoops(); - - // Requires "l1" to be a valid loop table index, and not "BasicBlock::NOT_IN_LOOP". - // Requires "l2" to be a valid loop table index, or else "BasicBlock::NOT_IN_LOOP". - // Returns true iff "l2" is not NOT_IN_LOOP, and "l1" contains "l2". - // A loop contains itself. - bool optLoopContains(unsigned l1, unsigned l2) const; - - // Returns the lpEntry for given preheader block of a loop - BasicBlock* optLoopEntry(BasicBlock* preHeader); - - // Updates the loop table by changing loop "loopInd", whose head is required - // to be "from", to be "to". Also performs this transformation for any - // loop nested in "loopInd" that shares the same head as "loopInd". - void optUpdateLoopHead(unsigned loopInd, BasicBlock* from, BasicBlock* to); - enum class RedirectBlockOption { DoNotChangePredLists, // do not modify pred lists @@ -7176,21 +6837,6 @@ class Compiler // Adds "elemType" to the set of modified array element types of "loop" and any parent loops. void AddModifiedElemTypeAllContainingLoops(FlowGraphNaturalLoop* loop, CORINFO_CLASS_HANDLE elemType); - // Returns true if 'block' is an entry block for any loop in 'optLoopTable' - bool optIsLoopEntry(BasicBlock* block) const; - - // The depth of the loop described by "lnum" (an index into the loop table.) (0 == top level) - unsigned optLoopDepth(unsigned lnum) const - { - assert(lnum < optLoopCount); - unsigned depth = 0; - while ((lnum = optLoopTable[lnum].lpParent) != BasicBlock::NOT_IN_LOOP) - { - ++depth; - } - return depth; - } - // Struct used in optInvertWhileLoop to count interesting constructs to boost the profitability score. struct OptInvertCountTreeInfoType { @@ -7217,27 +6863,8 @@ class Compiler unsigned* iterCount); protected: - struct isVarAssgDsc - { - GenTree* ivaSkip; - ALLVARSET_TP ivaMaskVal; // Set of variables assigned to. This is a set of all vars, not tracked vars. - unsigned ivaVar; // Variable we are interested in, or -1 - varRefKinds ivaMaskInd; // What kind of indirect assignments are there? - callInterf ivaMaskCall; // What kind of calls are there? - bool ivaMaskIncomplete; // Variables not representable in ivaMaskVal were assigned to. - }; - - bool optIsVarAssignedWithDesc(Statement* stmt, isVarAssgDsc* dsc); - - bool optIsVarAssigned(BasicBlock* beg, BasicBlock* end, GenTree* skip, unsigned var); - - bool optIsVarAssgLoop(unsigned lnum, unsigned var); - - bool optIsSetAssgLoop(unsigned lnum, ALLVARSET_VALARG_TP vars, varRefKinds inds = VR_NONE); - bool optNarrowTree(GenTree* tree, var_types srct, var_types dstt, ValueNumPair vnpNarrow, bool doit); -protected: // The following is the upper limit on how many expressions we'll keep track // of for the CSE analysis. // @@ -7423,8 +7050,6 @@ class Compiler void optOptimizeCSEs(); - static callInterf optCallInterf(GenTreeCall* call); - public: // VN based copy propagation. @@ -12217,7 +11842,6 @@ extern Histogram computeReachabilitySetsIterationTable; extern unsigned totalLoopMethods; // counts the total number of methods that have natural loops extern unsigned maxLoopsPerMethod; // counts the maximum number of loops a method has -extern unsigned totalLoopOverflows; // # of methods that identified more loops than we can represent extern unsigned totalLoopCount; // counts the total number of natural loops extern unsigned totalUnnatLoopCount; // counts the total number of (not-necessarily natural) loops extern unsigned totalUnnatLoopOverflows; // # of methods that identified more unnatural loops than we can represent diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index b00f3e6613921..a4cc4bcd13c08 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -3616,208 +3616,6 @@ inline void Compiler::optAssertionRemove(AssertionIndex index) } } -inline void Compiler::LoopDsc::VERIFY_lpIterTree() const -{ -#ifdef DEBUG - assert(lpFlags & LPFLG_ITER); - - // iterTree should be "lcl = lcl const" - - assert(lpIterTree->OperIs(GT_STORE_LCL_VAR)); - - const GenTree* value = lpIterTree->AsLclVar()->Data(); - assert(value->OperIs(GT_ADD, GT_SUB, GT_MUL, GT_RSH, GT_LSH)); - assert(value->AsOp()->gtOp1->OperGet() == GT_LCL_VAR); - assert(value->AsOp()->gtOp1->AsLclVar()->GetLclNum() == lpIterTree->AsLclVar()->GetLclNum()); - assert(value->AsOp()->gtOp2->OperGet() == GT_CNS_INT); -#endif -} - -//----------------------------------------------------------------------------- - -inline unsigned Compiler::LoopDsc::lpIterVar() const -{ - VERIFY_lpIterTree(); - return lpIterTree->AsLclVar()->GetLclNum(); -} - -//----------------------------------------------------------------------------- - -inline int Compiler::LoopDsc::lpIterConst() const -{ - VERIFY_lpIterTree(); - GenTree* value = lpIterTree->AsLclVar()->Data(); - return (int)value->AsOp()->gtOp2->AsIntCon()->gtIconVal; -} - -//----------------------------------------------------------------------------- - -inline genTreeOps Compiler::LoopDsc::lpIterOper() const -{ - VERIFY_lpIterTree(); - return lpIterTree->AsLclVar()->Data()->OperGet(); -} - -inline var_types Compiler::LoopDsc::lpIterOperType() const -{ - VERIFY_lpIterTree(); - - var_types type = lpIterTree->TypeGet(); - assert(genActualType(type) == TYP_INT); - - return type; -} - -inline void Compiler::LoopDsc::VERIFY_lpTestTree() const -{ -#ifdef DEBUG - assert(lpFlags & LPFLG_ITER); - assert(lpTestTree); - - genTreeOps oper = lpTestTree->OperGet(); - assert(GenTree::OperIsCompare(oper)); - - GenTree* iterator = nullptr; - GenTree* limit = nullptr; - if ((lpTestTree->AsOp()->gtOp2->gtOper == GT_LCL_VAR) && - (lpTestTree->AsOp()->gtOp2->gtFlags & GTF_VAR_ITERATOR) != 0) - { - iterator = lpTestTree->AsOp()->gtOp2; - limit = lpTestTree->AsOp()->gtOp1; - } - else if ((lpTestTree->AsOp()->gtOp1->gtOper == GT_LCL_VAR) && - (lpTestTree->AsOp()->gtOp1->gtFlags & GTF_VAR_ITERATOR) != 0) - { - iterator = lpTestTree->AsOp()->gtOp1; - limit = lpTestTree->AsOp()->gtOp2; - } - else - { - // one of the nodes has to be the iterator - assert(false); - } - - if (lpFlags & LPFLG_CONST_LIMIT) - { - assert(limit->OperIsConst()); - } - if (lpFlags & LPFLG_VAR_LIMIT) - { - assert(limit->OperGet() == GT_LCL_VAR); - } - if (lpFlags & LPFLG_ARRLEN_LIMIT) - { - assert(limit->OperGet() == GT_ARR_LENGTH); - } -#endif -} - -//----------------------------------------------------------------------------- - -inline bool Compiler::LoopDsc::lpIsReversed() const -{ - VERIFY_lpTestTree(); - return ((lpTestTree->AsOp()->gtOp2->gtOper == GT_LCL_VAR) && - (lpTestTree->AsOp()->gtOp2->gtFlags & GTF_VAR_ITERATOR) != 0); -} - -//----------------------------------------------------------------------------- - -inline genTreeOps Compiler::LoopDsc::lpTestOper() const -{ - VERIFY_lpTestTree(); - genTreeOps op = lpTestTree->OperGet(); - return lpIsReversed() ? GenTree::SwapRelop(op) : op; -} - -//----------------------------------------------------------------------------- - -inline bool Compiler::LoopDsc::lpIsIncreasingLoop() const -{ - // Increasing loop is the one that has "+=" increment operation and "< or <=" limit check. - bool isLessThanLimitCheck = GenTree::StaticOperIs(lpTestOper(), GT_LT, GT_LE); - return (isLessThanLimitCheck && - (((lpIterOper() == GT_ADD) && (lpIterConst() > 0)) || ((lpIterOper() == GT_SUB) && (lpIterConst() < 0)))); -} - -//----------------------------------------------------------------------------- - -inline bool Compiler::LoopDsc::lpIsDecreasingLoop() const -{ - // Decreasing loop is the one that has "-=" decrement operation and "> or >=" limit check. If the operation is - // "+=", make sure the constant is negative to give an effect of decrementing the iterator. - bool isGreaterThanLimitCheck = GenTree::StaticOperIs(lpTestOper(), GT_GT, GT_GE); - return (isGreaterThanLimitCheck && - (((lpIterOper() == GT_ADD) && (lpIterConst() < 0)) || ((lpIterOper() == GT_SUB) && (lpIterConst() > 0)))); -} - -//----------------------------------------------------------------------------- - -inline GenTree* Compiler::LoopDsc::lpIterator() const -{ - VERIFY_lpTestTree(); - - return lpIsReversed() ? lpTestTree->AsOp()->gtOp2 : lpTestTree->AsOp()->gtOp1; -} - -//----------------------------------------------------------------------------- - -inline GenTree* Compiler::LoopDsc::lpLimit() const -{ - VERIFY_lpTestTree(); - - return lpIsReversed() ? lpTestTree->AsOp()->gtOp1 : lpTestTree->AsOp()->gtOp2; -} - -//----------------------------------------------------------------------------- - -inline int Compiler::LoopDsc::lpConstLimit() const -{ - VERIFY_lpTestTree(); - assert(lpFlags & LPFLG_CONST_LIMIT); - - GenTree* limit = lpLimit(); - assert(limit->OperIsConst()); - return (int)limit->AsIntCon()->gtIconVal; -} - -//----------------------------------------------------------------------------- - -inline unsigned Compiler::LoopDsc::lpVarLimit() const -{ - VERIFY_lpTestTree(); - assert(lpFlags & LPFLG_VAR_LIMIT); - - GenTree* limit = lpLimit(); - assert(limit->OperGet() == GT_LCL_VAR); - return limit->AsLclVarCommon()->GetLclNum(); -} - -//----------------------------------------------------------------------------- - -inline bool Compiler::LoopDsc::lpArrLenLimit(Compiler* comp, ArrIndex* index) const -{ - VERIFY_lpTestTree(); - assert(lpFlags & LPFLG_ARRLEN_LIMIT); - - GenTree* limit = lpLimit(); - assert(limit->OperGet() == GT_ARR_LENGTH); - - // Check if we have a.length or a[i][j].length - if (limit->AsArrLen()->ArrRef()->gtOper == GT_LCL_VAR) - { - index->arrLcl = limit->AsArrLen()->ArrRef()->AsLclVarCommon()->GetLclNum(); - index->rank = 0; - return true; - } - // We have a[i].length, extract a[i] pattern. - else if (limit->AsArrLen()->ArrRef()->gtOper == GT_COMMA) - { - return comp->optReconstructArrIndex(limit->AsArrLen()->ArrRef(), index); - } - return false; -} - /* XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX diff --git a/src/coreclr/jit/fgbasic.cpp b/src/coreclr/jit/fgbasic.cpp index 6acdd220f20b0..3b768ab43e7d1 100644 --- a/src/coreclr/jit/fgbasic.cpp +++ b/src/coreclr/jit/fgbasic.cpp @@ -4924,17 +4924,6 @@ BasicBlock* Compiler::fgSplitBlockBeforeTree( assert(prevBb->KindIs(BBJ_ALWAYS) && prevBb->JumpsToNext() && prevBb->NextIs(block)); prevBb->SetFlags(BBF_NONE_QUIRK); - if (optLoopTableValid && prevBb->bbNatLoopNum != BasicBlock::NOT_IN_LOOP) - { - block->bbNatLoopNum = prevBb->bbNatLoopNum; - - // Update lpBottom after block split - if (optLoopTable[prevBb->bbNatLoopNum].lpBottom == prevBb) - { - optLoopTable[prevBb->bbNatLoopNum].lpBottom = block; - } - } - return block; } @@ -5355,9 +5344,6 @@ BasicBlock* Compiler::fgRemoveBlock(BasicBlock* block, bool unreachable) } #endif // FEATURE_EH_FUNCLETS - /* First update the loop table and bbWeights */ - optUpdateLoopsBeforeRemoveBlock(block, skipUnmarkLoop); - // Update successor block start IL offset, if empty predecessor // covers the immediately preceding range. if ((block->bbCodeOffsEnd == succBlock->bbCodeOffs) && (block->bbCodeOffs != BAD_IL_OFFSET)) @@ -5390,14 +5376,6 @@ BasicBlock* Compiler::fgRemoveBlock(BasicBlock* block, bool unreachable) { BasicBlock* predBlock = pred->getSourceBlock(); - /* Are we changing a loop backedge into a forward jump? */ - - if (block->isLoopHead() && (predBlock->bbNum >= block->bbNum) && (predBlock->bbNum <= succBlock->bbNum)) - { - /* First update the loop table and bbWeights */ - optUpdateLoopsBeforeRemoveBlock(predBlock); - } - /* If predBlock is a new predecessor, then add it to succBlock's predecessor's list. */ if (!predBlock->KindIs(BBJ_SWITCH)) @@ -5556,9 +5534,6 @@ BasicBlock* Compiler::fgConnectFallThrough(BasicBlock* bSrc, BasicBlock* bDst, b bSrc->SetFalseTarget(jmpBlk); fgAddRefPred(jmpBlk, bSrc, fgGetPredForBlock(bDst, bSrc)); - // Record the loop number in the new block - jmpBlk->bbNatLoopNum = bSrc->bbNatLoopNum; - // When adding a new jmpBlk we will set the bbWeight and bbFlags // if (fgHaveValidEdgeWeights && fgHaveProfileWeights()) @@ -6161,46 +6136,6 @@ BasicBlock* Compiler::fgRelocateEHRange(unsigned regionIndex, FG_RELOCATE_TYPE r return bLast; } -//------------------------------------------------------------------------ -// fgMightHaveLoop: return true if there is a possibility that the method has a loop (a back edge is present). -// This function doesn't depend on any previous loop computations, including predecessors. It looks for any -// lexical back edge to a block previously seen in a forward walk of the block list. -// -// As it walks all blocks and all successors of each block (including EH successors), it is not cheap. -// It returns as soon as any possible loop is discovered. -// -// Return Value: -// true if there might be a loop -// -bool Compiler::fgMightHaveLoop() -{ - // Don't use a BlockSet for this temporary bitset of blocks: we don't want to have to call EnsureBasicBlockEpoch() - // and potentially change the block epoch. - - BitVecTraits blockVecTraits(fgBBNumMax + 1, this); - BitVec blocksSeen(BitVecOps::MakeEmpty(&blockVecTraits)); - - for (BasicBlock* const block : Blocks()) - { - BitVecOps::AddElemD(&blockVecTraits, blocksSeen, block->bbNum); - - BasicBlockVisit result = block->VisitAllSuccs(this, [&](BasicBlock* succ) { - if (BitVecOps::IsMember(&blockVecTraits, blocksSeen, succ->bbNum)) - { - return BasicBlockVisit::Abort; - } - - return BasicBlockVisit::Continue; - }); - - if (result == BasicBlockVisit::Abort) - { - return true; - } - } - return false; -} - /***************************************************************************** * * Insert a BasicBlock before the given block. @@ -7072,58 +7007,6 @@ BasicBlock* Compiler::fgNewBBinRegionWorker(BBKinds jumpKind, /* If afterBlk falls through, we insert a jump around newBlk */ fgConnectFallThrough(afterBlk, newBlk->Next()); - // If the loop table is valid, add this block to the appropriate loop. - // Note we don't verify (via flow) that this block actually belongs - // to the loop, just that it is lexically within the span of blocks - // in the loop. - // - if (optLoopTableValid) - { - BasicBlock* const bbPrev = newBlk->Prev(); - BasicBlock* const bbNext = newBlk->Next(); - - if ((bbPrev != nullptr) && (bbNext != nullptr)) - { - BasicBlock::loopNumber const prevLoopNum = bbPrev->bbNatLoopNum; - BasicBlock::loopNumber const nextLoopNum = bbNext->bbNatLoopNum; - - if ((prevLoopNum != BasicBlock::NOT_IN_LOOP) && (nextLoopNum != BasicBlock::NOT_IN_LOOP)) - { - if (prevLoopNum == nextLoopNum) - { - newBlk->bbNatLoopNum = prevLoopNum; - } - else - { - BasicBlock::loopNumber const prevParentLoopNum = optLoopTable[prevLoopNum].lpParent; - BasicBlock::loopNumber const nextParentLoopNum = optLoopTable[nextLoopNum].lpParent; - - if (nextParentLoopNum == prevLoopNum) - { - // next is in child loop - newBlk->bbNatLoopNum = prevLoopNum; - } - else if (prevParentLoopNum == nextLoopNum) - { - // prev is in child loop - newBlk->bbNatLoopNum = nextLoopNum; - } - else - { - // next and prev are siblings - assert(prevParentLoopNum == nextParentLoopNum); - newBlk->bbNatLoopNum = prevParentLoopNum; - } - } - } - } - - if (newBlk->bbNatLoopNum != BasicBlock::NOT_IN_LOOP) - { - JITDUMP("Marked " FMT_BB " as lying within " FMT_LP "\n", newBlk->bbNum, newBlk->bbNatLoopNum); - } - } - #ifdef DEBUG fgVerifyHandlerTab(); #endif diff --git a/src/coreclr/jit/fgdiagnostic.cpp b/src/coreclr/jit/fgdiagnostic.cpp index c0f1ade1d077e..f1f1afd357bce 100644 --- a/src/coreclr/jit/fgdiagnostic.cpp +++ b/src/coreclr/jit/fgdiagnostic.cpp @@ -711,24 +711,19 @@ bool Compiler::fgDumpFlowGraph(Phases phase, PhasePosition pos) bool dontClose = false; #ifdef DEBUG - const bool createDotFile = JitConfig.JitDumpFgDot() != 0; - const bool includeEH = (JitConfig.JitDumpFgEH() != 0) && !compIsForInlining(); - const bool includeLoops = (JitConfig.JitDumpFgLoops() != 0) && !compIsForInlining(); - // The loop table is not well maintained after the optimization phases, but there is no single point at which - // it is declared invalid. For now, refuse to add loop information starting at the rationalize phase, to - // avoid asserts. - const bool includeOldLoops = - (JitConfig.JitDumpFgOldLoops() != 0) && !compIsForInlining() && (phase < PHASE_RATIONALIZE); + const bool createDotFile = JitConfig.JitDumpFgDot() != 0; + const bool includeEH = (JitConfig.JitDumpFgEH() != 0) && !compIsForInlining(); + const bool includeLoops = (JitConfig.JitDumpFgLoops() != 0) && !compIsForInlining(); const bool constrained = JitConfig.JitDumpFgConstrained() != 0; const bool useBlockId = JitConfig.JitDumpFgBlockID() != 0; const bool displayBlockFlags = JitConfig.JitDumpFgBlockFlags() != 0; #else // !DEBUG - const bool createDotFile = true; - const bool includeEH = false; - const bool includeLoops = false; - const bool constrained = true; - const bool useBlockId = false; - const bool displayBlockFlags = false; + const bool createDotFile = true; + const bool includeEH = false; + const bool includeLoops = false; + const bool constrained = true; + const bool useBlockId = false; + const bool displayBlockFlags = false; #endif // !DEBUG FILE* fgxFile = fgOpenFlowGraphFile(&dontClose, phase, pos, createDotFile ? "dot" : "fgx"); @@ -1203,7 +1198,7 @@ bool Compiler::fgDumpFlowGraph(Phases phase, PhasePosition pos) } } - if ((includeEH && (compHndBBtabCount > 0)) || (includeOldLoops && (optLoopCount > 0))) + if (includeEH && (compHndBBtabCount > 0)) { // Generate something like: // subgraph cluster_0 { @@ -1221,8 +1216,7 @@ bool Compiler::fgDumpFlowGraph(Phases phase, PhasePosition pos) // Thus, the subgraphs need to be nested to show the region nesting. // // The EH table is in order, top-to-bottom, most nested to least nested where - // there is a parent/child relationship. The loop table the opposite: it is - // in order from the least nested to most nested. + // there is a parent/child relationship. // // Build a region tree, collecting all the regions we want to display, // and then walk it to emit the regions. @@ -1235,7 +1229,6 @@ bool Compiler::fgDumpFlowGraph(Phases phase, PhasePosition pos) { Root, EH, - Loop }; private: @@ -1424,8 +1417,6 @@ bool Compiler::fgDumpFlowGraph(Phases phase, PhasePosition pos) return "Root"; case RegionType::EH: return "EH"; - case RegionType::Loop: - return "Loop"; default: return "UNKNOWN"; } @@ -1566,8 +1557,6 @@ bool Compiler::fgDumpFlowGraph(Phases phase, PhasePosition pos) { case RegionType::EH: return "red"; - case RegionType::Loop: - return "blue"; default: return "black"; } @@ -1703,46 +1692,6 @@ bool Compiler::fgDumpFlowGraph(Phases phase, PhasePosition pos) } } - // Add regions for the loops. Note that loops are assumed to be contiguous from `lpTop` to `lpBottom`. - - if (includeOldLoops) - { -#ifdef DEBUG - const bool displayLoopFlags = JitConfig.JitDumpFgLoopFlags() != 0; -#else // !DEBUG - const bool displayLoopFlags = false; -#endif // !DEBUG - - char name[30]; - for (unsigned loopNum = 0; loopNum < optLoopCount; loopNum++) - { - const LoopDsc& loop = optLoopTable[loopNum]; - if (loop.lpIsRemoved()) - { - continue; - } - - sprintf_s(name, sizeof(name), FMT_LP, loopNum); - - if (displayLoopFlags) - { - // Display a very few, useful, loop flags - strcat_s(name, sizeof(name), " ["); - if (loop.lpFlags & LoopFlags::LPFLG_ITER) - { - strcat_s(name, sizeof(name), "I"); - } - if (loop.lpFlags & LoopFlags::LPFLG_HAS_PREHEAD) - { - strcat_s(name, sizeof(name), "P"); - } - strcat_s(name, sizeof(name), "]"); - } - - rgnGraph.Insert(name, RegionGraph::RegionType::Loop, loop.lpTop, loop.lpBottom); - } - } - // All the regions have been added. Now, output them. DBEXEC(verbose, rgnGraph.Dump()); INDEBUG(rgnGraph.Verify()); @@ -1965,18 +1914,6 @@ void Compiler::fgTableDispBasicBlock(BasicBlock* block, int ibcColWidth /* = 0 * printf(" "); - // - // Display natural loop number - // - if (block->bbNatLoopNum == BasicBlock::NOT_IN_LOOP) - { - printf(" "); - } - else - { - printf("%2d ", block->bbNatLoopNum); - } - // // Display block IL range // @@ -2347,7 +2284,7 @@ void Compiler::fgDispBasicBlocks(BasicBlock* firstBlock, BasicBlock* lastBlock, padWidth, "------------", ibcColWidth, "------------", maxBlockNumWidth, "----"); - printf("BBnum %*sBBid ref try hnd %s weight %*s%s lp [IL range] [jump]%*s [EH region] [flags]\n", + printf("BBnum %*sBBid ref try hnd %s weight %*s%s [IL range] [jump]%*s [EH region] [flags]\n", padWidth, "", (fgPredsComputed ? "preds " : " "), @@ -3208,7 +3145,7 @@ void Compiler::fgDebugCheckBBlist(bool checkBBNum /* = false */, bool checkBBRef #ifndef JIT32_GCENCODER copiedForGenericsCtxt = ((info.compMethodInfo->options & CORINFO_GENERICS_CTXT_FROM_THIS) != 0); #else // JIT32_GCENCODER - copiedForGenericsCtxt = false; + copiedForGenericsCtxt = false; #endif // JIT32_GCENCODER // This if only in support of the noway_asserts it contains. @@ -4720,429 +4657,22 @@ void Compiler::fgDebugCheckSsa() } //------------------------------------------------------------------------------ -// fgDebugCheckLoopTable: checks that the loop table is valid. -// - If the method has natural loops, the loop table is not null -// - Loop `top` must come before `bottom`. -// - Loop `entry` must be between `top` and `bottom`. -// - Children loops of a loop are disjoint. -// - All basic blocks with loop numbers set have a corresponding loop in the table -// - All basic blocks without a loop number are not in a loop -// - All parents of the loop with the block contain that block -// - If optLoopsRequirePreHeaders is true, the loop has a pre-header -// - If the loop has a pre-header, it is valid -// - The loop flags are valid -// - no loop shares `top` with any of its children -// - no loop shares `bottom` with the header of any of its siblings -// - no top-entry loop has a predecessor that comes from outside the loop other than from lpHead +// fgDebugCheckLoops: Checks that all loops are canonicalized as expected. // -void Compiler::fgDebugCheckLoopTable() +void Compiler::fgDebugCheckLoops() { - if ((m_loops != nullptr) && optLoopsRequirePreHeaders) + if (m_loops == nullptr) { - for (FlowGraphNaturalLoop* loop : m_loops->InReversePostOrder()) - { - assert(loop->EntryEdges().size() == 1); - assert(loop->EntryEdge(0)->getSourceBlock()->KindIs(BBJ_ALWAYS)); - } - } - -#ifdef DEBUG - if (!optLoopTableValid) - { - JITDUMP("*************** In fgDebugCheckLoopTable: loop table not valid\n"); return; } - JITDUMP("*************** In fgDebugCheckLoopTable\n"); -#endif // DEBUG - - if (optLoopCount > 0) - { - assert(optLoopTable != nullptr); - } - - // Build a mapping from existing block list number (bbNum) to the block number it would be after the - // blocks are renumbered. This allows making asserts about the relative ordering of blocks using block number - // without actually renumbering the blocks, which would affect non-DEBUG code paths. Note that there may be - // `blockNumMap[bbNum] == 0` if the `bbNum` block was deleted and blocks haven't been renumbered since - // the deletion. - - unsigned bbNumMax = fgBBNumMax; - - // blockNumMap[old block number] => new block number - size_t blockNumBytes = (bbNumMax + 1) * sizeof(unsigned); - unsigned* blockNumMap = (unsigned*)_alloca(blockNumBytes); - memset(blockNumMap, 0, blockNumBytes); - - unsigned newBBnum = 1; - for (BasicBlock* const block : Blocks()) - { - if (!block->HasFlag(BBF_REMOVED)) - { - assert(1 <= block->bbNum && block->bbNum <= bbNumMax); - assert(blockNumMap[block->bbNum] == 0); // If this fails, we have two blocks with the same block number. - blockNumMap[block->bbNum] = newBBnum++; - } - } - - struct MappedChecks + if (optLoopsRequirePreHeaders) { - static bool lpWellFormed(const unsigned* blockNumMap, const LoopDsc* loop) - { - return (blockNumMap[loop->lpTop->bbNum] <= blockNumMap[loop->lpEntry->bbNum]) && - (blockNumMap[loop->lpEntry->bbNum] <= blockNumMap[loop->lpBottom->bbNum]) && - ((blockNumMap[loop->lpHead->bbNum] < blockNumMap[loop->lpTop->bbNum]) || - (blockNumMap[loop->lpHead->bbNum] > blockNumMap[loop->lpBottom->bbNum])); - } - - static bool lpContains(const unsigned* blockNumMap, const LoopDsc* loop, const BasicBlock* blk) - { - return (blockNumMap[loop->lpTop->bbNum] <= blockNumMap[blk->bbNum]) && - (blockNumMap[blk->bbNum] <= blockNumMap[loop->lpBottom->bbNum]); - } - static bool lpContains(const unsigned* blockNumMap, - const LoopDsc* loop, - const BasicBlock* top, - const BasicBlock* bottom) - { - return (blockNumMap[loop->lpTop->bbNum] <= blockNumMap[top->bbNum]) && - (blockNumMap[bottom->bbNum] < blockNumMap[loop->lpBottom->bbNum]); - } - static bool lpContains(const unsigned* blockNumMap, const LoopDsc* loop, const LoopDsc& lp2) - { - return lpContains(blockNumMap, loop, lp2.lpTop, lp2.lpBottom); - } - - static bool lpContainedBy(const unsigned* blockNumMap, - const LoopDsc* loop, - const BasicBlock* top, - const BasicBlock* bottom) - { - return (blockNumMap[top->bbNum] <= blockNumMap[loop->lpTop->bbNum]) && - (blockNumMap[loop->lpBottom->bbNum] < blockNumMap[bottom->bbNum]); - } - static bool lpContainedBy(const unsigned* blockNumMap, const LoopDsc* loop, const LoopDsc& lp2) - { - return lpContainedBy(blockNumMap, loop, lp2.lpTop, lp2.lpBottom); - } - - static bool lpDisjoint(const unsigned* blockNumMap, - const LoopDsc* loop, - const BasicBlock* top, - const BasicBlock* bottom) - { - return (blockNumMap[bottom->bbNum] < blockNumMap[loop->lpTop->bbNum]) || - (blockNumMap[loop->lpBottom->bbNum] < blockNumMap[top->bbNum]); - } - static bool lpDisjoint(const unsigned* blockNumMap, const LoopDsc* loop, const LoopDsc& lp2) - { - return lpDisjoint(blockNumMap, loop, lp2.lpTop, lp2.lpBottom); - } - - // Like Disjoint, but also checks lpHead - // - static bool lpFullyDisjoint(const unsigned* blockNumMap, const LoopDsc* loop, const LoopDsc& lp2) - { - return lpDisjoint(blockNumMap, loop, lp2.lpTop, lp2.lpBottom) && - !lpContains(blockNumMap, loop, lp2.lpHead) && !lpContains(blockNumMap, &lp2, loop->lpHead); - } - - // If a top-entry loop, lpHead must be only non-loop pred of lpTop - static bool lpHasWellFormedBackedges(const unsigned* blockNumMap, const LoopDsc* loop) - { - if (loop->lpTop != loop->lpEntry) - { - // not top-entry, assume ok for now - return true; - } - - bool foundHead = false; - for (BasicBlock* const pred : loop->lpTop->PredBlocks()) - { - if (pred == loop->lpHead) - { - foundHead = true; - continue; - } - - if (!lpContains(blockNumMap, loop, pred)) - { - return false; - } - } - - return foundHead; - } - }; - - // Check the loop table itself. - - int preHeaderCount = 0; - - for (unsigned i = 0; i < optLoopCount; i++) - { - const LoopDsc& loop = optLoopTable[i]; - - // Ignore removed loops - if (loop.lpIsRemoved()) - { - continue; - } - - assert(loop.lpHead != nullptr); - assert(loop.lpTop != nullptr); - assert(loop.lpEntry != nullptr); - assert(loop.lpBottom != nullptr); - - assert(MappedChecks::lpWellFormed(blockNumMap, &loop)); - assert(MappedChecks::lpHasWellFormedBackedges(blockNumMap, &loop)); - - if (loop.lpExitCnt == 1) - { - assert(loop.lpExit != nullptr); - assert(MappedChecks::lpContains(blockNumMap, &loop, loop.lpExit)); - } - else - { - assert(loop.lpExit == nullptr); - } - - if (loop.lpParent == BasicBlock::NOT_IN_LOOP) - { - // This is a top-level loop. - - // Verify all top-level loops are fully disjoint. We don't have a list of just these (such as a - // top-level pseudo-loop entry with a list of all top-level lists), so we have to iterate - // over the entire loop table. - for (unsigned j = 0; j < optLoopCount; j++) - { - if (i == j) - { - // Don't compare against ourselves. - continue; - } - const LoopDsc& otherLoop = optLoopTable[j]; - if (otherLoop.lpIsRemoved()) - { - continue; - } - if (otherLoop.lpParent != BasicBlock::NOT_IN_LOOP) - { - // Only consider top-level loops - continue; - } - assert(MappedChecks::lpFullyDisjoint(blockNumMap, &loop, otherLoop)); - } - } - else - { - // This is not a top-level loop - - assert(loop.lpParent != BasicBlock::NOT_IN_LOOP); - assert(loop.lpParent < optLoopCount); - assert(loop.lpParent < i); // outer loops come before inner loops in the table - - const LoopDsc& parentLoop = optLoopTable[loop.lpParent]; - assert(!parentLoop.lpIsRemoved()); // don't allow removed parent loop? - - // Either there is no sibling or it should not be marked REMOVED. - assert((loop.lpSibling == BasicBlock::NOT_IN_LOOP) || !optLoopTable[loop.lpSibling].lpIsRemoved()); - - // Either there is no child or it should not be marked REMOVED. - assert((loop.lpChild == BasicBlock::NOT_IN_LOOP) || !optLoopTable[loop.lpChild].lpIsRemoved()); - - assert(MappedChecks::lpContainedBy(blockNumMap, &loop, optLoopTable[loop.lpParent])); - } - - if (loop.lpChild != BasicBlock::NOT_IN_LOOP) - { - // Verify all child loops are contained in the parent loop. - for (unsigned child = loop.lpChild; // - child != BasicBlock::NOT_IN_LOOP; // - child = optLoopTable[child].lpSibling) - { - assert(child < optLoopCount); - assert(i < child); // outer loops come before inner loops in the table - const LoopDsc& childLoop = optLoopTable[child]; - assert(!childLoop.lpIsRemoved()); - assert(MappedChecks::lpContains(blockNumMap, &loop, childLoop)); - assert(childLoop.lpParent == i); - } - - // Verify all child loops are fully disjoint. - for (unsigned child = loop.lpChild; // - child != BasicBlock::NOT_IN_LOOP; // - child = optLoopTable[child].lpSibling) - { - const LoopDsc& childLoop = optLoopTable[child]; - assert(!childLoop.lpIsRemoved()); - for (unsigned child2 = optLoopTable[child].lpSibling; // - child2 != BasicBlock::NOT_IN_LOOP; // - child2 = optLoopTable[child2].lpSibling) - { - const LoopDsc& child2Loop = optLoopTable[child2]; - assert(!child2Loop.lpIsRemoved()); - assert(MappedChecks::lpFullyDisjoint(blockNumMap, &childLoop, child2Loop)); - } - } - - // Verify no child shares lpTop with its parent. - for (unsigned child = loop.lpChild; // - child != BasicBlock::NOT_IN_LOOP; // - child = optLoopTable[child].lpSibling) - { - const LoopDsc& childLoop = optLoopTable[child]; - assert(!childLoop.lpIsRemoved()); - assert(loop.lpTop != childLoop.lpTop); - } - } - - if (optLoopsRequirePreHeaders) - { - assert((loop.lpFlags & LPFLG_HAS_PREHEAD) != 0); - } - - // If the loop has a pre-header, ensure the pre-header form is correct. - if ((loop.lpFlags & LPFLG_HAS_PREHEAD) != 0) - { - ++preHeaderCount; - - BasicBlock* h = loop.lpHead; - assert(h->HasFlag(BBF_LOOP_PREHEADER)); - - // The pre-header can only be BBJ_ALWAYS and must enter the loop. - BasicBlock* e = loop.lpEntry; - assert(h->KindIs(BBJ_ALWAYS)); - assert(h->TargetIs(e)); - - // The entry block has a single non-loop predecessor, and it is the pre-header. - for (BasicBlock* const predBlock : e->PredBlocks()) - { - if (predBlock != h) - { - assert(MappedChecks::lpContains(blockNumMap, &loop, predBlock)); - } - } - - loop.lpValidatePreHeader(); - } - - // Check the flags. - // Note that the various limit flags are only used when LPFLG_ITER is set, but they are set first, - // separately, and only if everything works out is LPFLG_ITER set. If LPFLG_ITER is NOT set, the - // individual flags are not un-set (arguably, they should be). - - // Only one of the `limit` flags can be set. (Note that LPFLG_SIMD_LIMIT is a "sub-flag" that can be - // set when LPFLG_CONST_LIMIT is set.) - assert(genCountBits((unsigned)(loop.lpFlags & (LPFLG_VAR_LIMIT | LPFLG_CONST_LIMIT | LPFLG_ARRLEN_LIMIT))) <= - 1); - - // LPFLG_SIMD_LIMIT can only be set if LPFLG_CONST_LIMIT is set. - if (loop.lpFlags & LPFLG_SIMD_LIMIT) - { - assert(loop.lpFlags & LPFLG_CONST_LIMIT); - } - - if (loop.lpFlags & LPFLG_CONST_INIT) - { - assert(loop.lpInitBlock != nullptr); - } - - if (loop.lpFlags & LPFLG_ITER) - { - loop.VERIFY_lpIterTree(); - loop.VERIFY_lpTestTree(); - } - - // If we have dominators, we check more things: - // 1. The pre-header dominates the entry (if pre-headers are required). - // 2. The entry dominates the exit. - // 3. The IDom tree from the exit reaches the entry. - if (m_domTree != nullptr) - { - if (optLoopsRequirePreHeaders) - { - assert(m_domTree->Dominates(loop.lpHead, loop.lpEntry)); - } - - if (loop.lpExitCnt == 1) - { - assert(loop.lpExit != nullptr); - assert(m_domTree->Dominates(loop.lpEntry, loop.lpExit)); - - BasicBlock* cur = loop.lpExit; - while ((cur != nullptr) && (cur != loop.lpEntry)) - { - assert(m_domTree->Dominates(cur, loop.lpExit)); - cur = cur->bbIDom; - } - assert(cur == loop.lpEntry); // We must be able to reach the entry from the exit via the IDom tree. - } - } - } - - // Check basic blocks for loop annotations. - - for (BasicBlock* const block : Blocks()) - { - if (optLoopCount == 0) - { - assert(block->bbNatLoopNum == BasicBlock::NOT_IN_LOOP); - continue; - } - - // Walk the loop table and find the first loop that contains our block. - // It should be the innermost one. - int loopNum = BasicBlock::NOT_IN_LOOP; - for (int i = optLoopCount - 1; i >= 0; i--) - { - // Ignore removed loops - if (optLoopTable[i].lpIsRemoved()) - { - continue; - } - // Does this loop contain our block? - if (MappedChecks::lpContains(blockNumMap, &optLoopTable[i], block)) - { - loopNum = i; - break; - } - } - - // If there is at least one loop that contains this block... - if (loopNum != BasicBlock::NOT_IN_LOOP) - { - // ...it must be the one pointed to by bbNatLoopNum. - assert(block->bbNatLoopNum == loopNum); - - // TODO: We might want the following assert, but there are cases where we don't move all - // return blocks out of the loop. - // Return blocks are not allowed inside a loop; they should have been moved elsewhere. - // assert(!block->KindIs(BBJ_RETURN)); - } - else - { - // Otherwise, this block should not point to a loop. - assert(block->bbNatLoopNum == BasicBlock::NOT_IN_LOOP); - } - - // All loops that contain the innermost loop with this block must also contain this block. - while (loopNum != BasicBlock::NOT_IN_LOOP) - { - assert(MappedChecks::lpContains(blockNumMap, &optLoopTable[loopNum], block)); - loopNum = optLoopTable[loopNum].lpParent; - } - - if (block->HasFlag(BBF_LOOP_PREHEADER)) + for (FlowGraphNaturalLoop* loop : m_loops->InReversePostOrder()) { - // Note that the bbNatLoopNum will not point to the loop where this is a pre-header, since bbNatLoopNum - // is only set on the blocks from `top` to `bottom`, and `head` is outside that. - --preHeaderCount; + assert(loop->EntryEdges().size() == 1); + assert(loop->EntryEdge(0)->getSourceBlock()->KindIs(BBJ_ALWAYS)); } } - - // Verify that the number of loops marked as having pre-headers is the same as the number of blocks - // with the pre-header flag set. - assert(preHeaderCount == 0); } //------------------------------------------------------------------------------ diff --git a/src/coreclr/jit/fgopt.cpp b/src/coreclr/jit/fgopt.cpp index 16ef174b53c63..25afba806486a 100644 --- a/src/coreclr/jit/fgopt.cpp +++ b/src/coreclr/jit/fgopt.cpp @@ -1356,11 +1356,6 @@ void Compiler::fgCompactBlocks(BasicBlock* block, BasicBlock* bNext) block->RemoveFlags(BBF_NONE_QUIRK); } - if (optLoopTableValid) - { - fgUpdateLoopsAfterCompacting(block, bNext); - } - #if DEBUG if (verbose && 0) { @@ -1378,66 +1373,6 @@ void Compiler::fgCompactBlocks(BasicBlock* block, BasicBlock* bNext) #endif // DEBUG } -//------------------------------------------------------------- -// fgUpdateLoopsAfterCompacting: Update the loop table after block compaction. -// -// Arguments: -// block - target of compaction. -// bNext - bbNext of `block`. This block has been removed. -// -void Compiler::fgUpdateLoopsAfterCompacting(BasicBlock* block, BasicBlock* bNext) -{ - /* Check if the removed block is not part the loop table */ - noway_assert(bNext); - - for (unsigned loopNum = 0; loopNum < optLoopCount; loopNum++) - { - /* Some loops may have been already removed by - * loop unrolling or conditional folding */ - - if (optLoopTable[loopNum].lpIsRemoved()) - { - continue; - } - - /* Check the loop head (i.e. the block preceding the loop) */ - - if (optLoopTable[loopNum].lpHead == bNext) - { - optLoopTable[loopNum].lpHead = block; - } - - /* Check the loop bottom */ - - if (optLoopTable[loopNum].lpBottom == bNext) - { - optLoopTable[loopNum].lpBottom = block; - } - - /* Check the loop exit */ - - if (optLoopTable[loopNum].lpExit == bNext) - { - noway_assert(optLoopTable[loopNum].lpExitCnt == 1); - optLoopTable[loopNum].lpExit = block; - } - - /* Check the loop entry */ - - if (optLoopTable[loopNum].lpEntry == bNext) - { - optLoopTable[loopNum].lpEntry = block; - } - - /* Check the loop top */ - - if (optLoopTable[loopNum].lpTop == bNext) - { - optLoopTable[loopNum].lpTop = block; - } - } -} - //------------------------------------------------------------- // fgUnreachableBlock: Remove a block when it is unreachable. // @@ -1500,9 +1435,6 @@ void Compiler::fgUnreachableBlock(BasicBlock* block) noway_assert(block->bbStmtList == nullptr); } - // Next update the loop table and bbWeights - optUpdateLoopsBeforeRemoveBlock(block); - // Mark the block as removed block->SetFlags(BBF_REMOVED); @@ -4798,10 +4730,6 @@ PhaseStatus Compiler::fgUpdateFlowGraphPhase() constexpr bool isPhase = true; const bool madeChanges = fgUpdateFlowGraph(doTailDup, isPhase); - // The loop table is no longer valid. - optLoopTableValid = false; - optLoopsRequirePreHeaders = false; - return madeChanges ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING; } @@ -5157,12 +5085,6 @@ bool Compiler::fgUpdateFlowGraph(bool doTailDuplication, bool isPhase) /* Mark the block as removed */ bNext->SetFlags(BBF_REMOVED); - if (optLoopTableValid) - { - // Update the loop table if we removed the bottom of a loop, for example. - fgUpdateLoopsAfterCompacting(block, bNext); - } - // If this is the first Cold basic block update fgFirstColdBlock if (bNext->IsFirstColdBlock(this)) { diff --git a/src/coreclr/jit/flowgraph.cpp b/src/coreclr/jit/flowgraph.cpp index e65d161b8813a..79ee2e97881ed 100644 --- a/src/coreclr/jit/flowgraph.cpp +++ b/src/coreclr/jit/flowgraph.cpp @@ -266,16 +266,8 @@ BasicBlock* Compiler::fgCreateGCPoll(GCPollType pollType, BasicBlock* block) // I want to create: // top -> poll -> bottom (lexically) // so that we jump over poll to get to bottom. - BasicBlock* top = block; - unsigned char lpIndexFallThrough = BasicBlock::NOT_IN_LOOP; - - BBKinds oldJumpKind = top->GetKind(); - unsigned char lpIndex = top->bbNatLoopNum; - - if (oldJumpKind == BBJ_COND) - { - lpIndexFallThrough = top->GetFalseTarget()->bbNatLoopNum; - } + BasicBlock* top = block; + BBKinds oldJumpKind = top->GetKind(); BasicBlock* poll = fgNewBBafter(BBJ_ALWAYS, top, true); bottom = fgNewBBafter(top->GetKind(), poll, true); @@ -302,24 +294,6 @@ BasicBlock* Compiler::fgCreateGCPoll(GCPollType pollType, BasicBlock* block) // Mark Poll as rarely run. poll->bbSetRunRarely(); - if (optLoopTableValid) - { - poll->bbNatLoopNum = lpIndex; // Set the bbNatLoopNum in case we are in a loop - - bottom->bbNatLoopNum = lpIndex; // Set the bbNatLoopNum in case we are in a loop - if (lpIndex != BasicBlock::NOT_IN_LOOP) - { - // Set the new lpBottom in the natural loop table - optLoopTable[lpIndex].lpBottom = bottom; - } - - if (lpIndexFallThrough != BasicBlock::NOT_IN_LOOP) - { - // Set the new lpHead in the natural loop table - optLoopTable[lpIndexFallThrough].lpHead = bottom; - } - } - // Add the GC_CALL node to Poll. Statement* pollStmt = fgNewStmtAtEnd(poll, call); if (fgNodeThreading != NodeThreading::None) @@ -4568,24 +4542,13 @@ void FlowGraphNaturalLoop::Dump(FlowGraphNaturalLoop* loop) return; } - // Display: LOOP# (old LOOP#) / header / parent loop# / blocks / entry edges / exit edges / back edges + // Display: LOOP# / header / parent loop# / blocks / entry edges / exit edges / back edges // Blocks can compacted be "[top .. bottom]" if lexically adjacent and no non-loop blocks in // the range. Otherwise, print a verbose list of blocks. Compiler* comp = loop->GetDfsTree()->GetCompiler(); printf(FMT_LP, loop->GetIndex()); - // We might want to print out the old loop number using something like: - // - // if (comp->m_newToOldLoop[loop->GetIndex()] != nullptr) - // { - // printf(" (old: " FMT_LP ")", (unsigned)(comp->m_newToOldLoop[loop->GetIndex()] - comp->optLoopTable)); - // } - // - // However, not all callers of FlowGraphNaturalLoops::Find update m_newToOldLoop -- only - // Compiler::optFindNewLoops() does that. This dumper should work with any construction of - // FlowGraphNaturalLoops. - printf(" header: " FMT_BB, loop->GetHeader()->bbNum); if (loop->GetParent() != nullptr) { @@ -4786,7 +4749,7 @@ void FlowGraphNaturalLoop::Dump(FlowGraphNaturalLoop* loop) /* static */ void FlowGraphNaturalLoops::Dump(FlowGraphNaturalLoops* loops) { - printf("\n*************** (New) Natural loop graph\n"); + printf("\n*************** Natural loop graph\n"); if (loops == nullptr) { diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index b8f109c68b43a..ef4dd9be14691 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -453,7 +453,6 @@ enum GenTreeFlags : unsigned int GTF_LIVENESS_MASK = GTF_VAR_DEF | GTF_VAR_USEASG | GTF_VAR_DEATH_MASK, - GTF_VAR_ITERATOR = 0x01000000, // GT_LCL_VAR -- this is a iterator reference in the loop condition GTF_VAR_MOREUSES = 0x00800000, // GT_LCL_VAR -- this node has additional uses, for example due to cloning GTF_VAR_CONTEXT = 0x00400000, // GT_LCL_VAR -- this node is part of a runtime lookup GTF_VAR_EXPLICIT_INIT = 0x00200000, // GT_LCL_VAR -- this node is an "explicit init" store. Valid until rationalization. diff --git a/src/coreclr/jit/helperexpansion.cpp b/src/coreclr/jit/helperexpansion.cpp index ac2a65b33ef95..ad3b20bc7ecc6 100644 --- a/src/coreclr/jit/helperexpansion.cpp +++ b/src/coreclr/jit/helperexpansion.cpp @@ -450,17 +450,6 @@ bool Compiler::fgExpandRuntimeLookupsForCall(BasicBlock** pBlock, Statement* stm fallbackBb->inheritWeightPercentage(nullcheckBb, 20); } - // - // Update loop info - // - nullcheckBb->bbNatLoopNum = prevBb->bbNatLoopNum; - fastPathBb->bbNatLoopNum = prevBb->bbNatLoopNum; - fallbackBb->bbNatLoopNum = prevBb->bbNatLoopNum; - if (needsSizeCheck) - { - sizeCheckBb->bbNatLoopNum = prevBb->bbNatLoopNum; - } - // All blocks are expected to be in the same EH region assert(BasicBlock::sameEHRegion(prevBb, block)); assert(BasicBlock::sameEHRegion(prevBb, nullcheckBb)); @@ -694,13 +683,6 @@ bool Compiler::fgExpandThreadLocalAccessForCallNativeAOT(BasicBlock** pBlock, St fgAddRefPred(tlsRootNullCondBB, prevBb); prevBb->SetTarget(tlsRootNullCondBB); - // - // Update loop info if loop table is known to be valid - // - tlsRootNullCondBB->bbNatLoopNum = prevBb->bbNatLoopNum; - fastPathBb->bbNatLoopNum = prevBb->bbNatLoopNum; - fallbackBb->bbNatLoopNum = prevBb->bbNatLoopNum; - // All blocks are expected to be in the same EH region assert(BasicBlock::sameEHRegion(prevBb, block)); assert(BasicBlock::sameEHRegion(prevBb, tlsRootNullCondBB)); @@ -1080,14 +1062,6 @@ bool Compiler::fgExpandThreadLocalAccessForCall(BasicBlock** pBlock, Statement* // fallback will just execute first time fallbackBb->bbSetRunRarely(); - // - // Update loop info if loop table is known to be valid - // - maxThreadStaticBlocksCondBB->bbNatLoopNum = prevBb->bbNatLoopNum; - threadStaticBlockNullCondBB->bbNatLoopNum = prevBb->bbNatLoopNum; - fastPathBb->bbNatLoopNum = prevBb->bbNatLoopNum; - fallbackBb->bbNatLoopNum = prevBb->bbNatLoopNum; - // All blocks are expected to be in the same EH region assert(BasicBlock::sameEHRegion(prevBb, block)); assert(BasicBlock::sameEHRegion(prevBb, maxThreadStaticBlocksCondBB)); @@ -1449,13 +1423,6 @@ bool Compiler::fgExpandStaticInitForCall(BasicBlock** pBlock, Statement* stmt, G isInitedBb->inheritWeight(prevBb); helperCallBb->bbSetRunRarely(); - // - // Update loop info if loop table is known to be valid - // - - isInitedBb->bbNatLoopNum = prevBb->bbNatLoopNum; - helperCallBb->bbNatLoopNum = prevBb->bbNatLoopNum; - // All blocks are expected to be in the same EH region assert(BasicBlock::sameEHRegion(prevBb, block)); assert(BasicBlock::sameEHRegion(prevBb, isInitedBb)); @@ -1794,12 +1761,6 @@ bool Compiler::fgVNBasedIntrinsicExpansionForCall_ReadUtf8(BasicBlock** pBlock, fastpathBb->inheritWeight(lengthCheckBb); block->inheritWeight(prevBb); - // - // Update bbNatLoopNum for all new blocks - // - lengthCheckBb->bbNatLoopNum = prevBb->bbNatLoopNum; - fastpathBb->bbNatLoopNum = prevBb->bbNatLoopNum; - // All blocks are expected to be in the same EH region assert(BasicBlock::sameEHRegion(prevBb, block)); assert(BasicBlock::sameEHRegion(prevBb, lengthCheckBb)); @@ -2101,12 +2062,8 @@ bool Compiler::fgLateCastExpansionForCall(BasicBlock** pBlock, Statement* stmt, lastBb->inheritWeight(firstBb); // - // Update bbNatLoopNum for all new blocks and validate EH regions + // Validate EH regions // - nullcheckBb->bbNatLoopNum = firstBb->bbNatLoopNum; - fallbackBb->bbNatLoopNum = firstBb->bbNatLoopNum; - typeCheckBb->bbNatLoopNum = firstBb->bbNatLoopNum; - typeCheckSucceedBb->bbNatLoopNum = firstBb->bbNatLoopNum; assert(BasicBlock::sameEHRegion(firstBb, lastBb)); assert(BasicBlock::sameEHRegion(firstBb, nullcheckBb)); assert(BasicBlock::sameEHRegion(firstBb, fallbackBb)); diff --git a/src/coreclr/jit/jitconfigvalues.h b/src/coreclr/jit/jitconfigvalues.h index 82c1a1a0697ae..50afc5c7cf1c0 100644 --- a/src/coreclr/jit/jitconfigvalues.h +++ b/src/coreclr/jit/jitconfigvalues.h @@ -220,11 +220,9 @@ CONFIG_STRING(JitDumpFgPhase, W("JitDumpFgPhase")) // Phase-based Xml/Dot flowgr // phases CONFIG_STRING(JitDumpFgPrePhase, W("JitDumpFgPrePhase")) // Same as JitDumpFgPhase, but specifies to dump pre-phase, not post-phase. -CONFIG_INTEGER(JitDumpFgDot, W("JitDumpFgDot"), 1) // 0 == dump XML format; non-zero == dump DOT format -CONFIG_INTEGER(JitDumpFgEH, W("JitDumpFgEH"), 0) // 0 == no EH regions; non-zero == include EH regions -CONFIG_INTEGER(JitDumpFgLoops, W("JitDumpFgLoops"), 0) // 0 == no loop regions; non-zero == include loop regions -CONFIG_INTEGER(JitDumpFgOldLoops, W("JitDumpFgOldLoops"), 0) // 0 == no old loop regions; non-zero == include old loop - // regions +CONFIG_INTEGER(JitDumpFgDot, W("JitDumpFgDot"), 1) // 0 == dump XML format; non-zero == dump DOT format +CONFIG_INTEGER(JitDumpFgEH, W("JitDumpFgEH"), 0) // 0 == no EH regions; non-zero == include EH regions +CONFIG_INTEGER(JitDumpFgLoops, W("JitDumpFgLoops"), 0) // 0 == no loop regions; non-zero == include loop regions CONFIG_INTEGER(JitDumpFgConstrained, W("JitDumpFgConstrained"), 1) // 0 == don't constrain to mostly linear layout; // non-zero == force mostly lexical block diff --git a/src/coreclr/jit/loopcloning.cpp b/src/coreclr/jit/loopcloning.cpp index fc9703feafcad..730eb91738f67 100644 --- a/src/coreclr/jit/loopcloning.cpp +++ b/src/coreclr/jit/loopcloning.cpp @@ -862,7 +862,6 @@ BasicBlock* LoopCloneContext::CondToStmtInBlock(Compiler* { BasicBlock* newBlk = comp->fgNewBBafter(BBJ_COND, insertAfter, /*extendRegion*/ true, slowPreheader); newBlk->inheritWeight(insertAfter); - newBlk->bbNatLoopNum = insertAfter->bbNatLoopNum; JITDUMP("Adding " FMT_BB " -> " FMT_BB "\n", newBlk->bbNum, newBlk->GetTrueTarget()->bbNum); comp->fgAddRefPred(newBlk->GetTrueTarget(), newBlk); @@ -897,7 +896,6 @@ BasicBlock* LoopCloneContext::CondToStmtInBlock(Compiler* { BasicBlock* newBlk = comp->fgNewBBafter(BBJ_COND, insertAfter, /*extendRegion*/ true, slowPreheader); newBlk->inheritWeight(insertAfter); - newBlk->bbNatLoopNum = insertAfter->bbNatLoopNum; JITDUMP("Adding " FMT_BB " -> " FMT_BB "\n", newBlk->bbNum, newBlk->GetTrueTarget()->bbNum); comp->fgAddRefPred(newBlk->GetTrueTarget(), newBlk); @@ -2098,9 +2096,11 @@ void Compiler::optCloneLoop(FlowGraphNaturalLoop* loop, LoopCloneContext* contex // TODO: scale the pred edges of `blk`? // If the loop we're cloning contains nested loops, we need to clear the pre-header bit on - // any nested loop pre-header blocks, since they will no longer be loop pre-headers. (This is because - // we don't add the slow loop or its child loops to the loop table. It would be simplest to - // just re-build the loop table if we want to enable loop optimization of the slow path loops.) + // any nested loop pre-header blocks, since they will no longer be loop pre-headers. + // + // TODO-Cleanup: BBF_LOOP_PREHEADER can be removed; we do not attempt + // to keep it up to date anymore when we do FG changes. + // if (newBlk->HasFlag(BBF_LOOP_PREHEADER)) { JITDUMP("Removing BBF_LOOP_PREHEADER flag from nested cloned loop block " FMT_BB "\n", newBlk->bbNum); diff --git a/src/coreclr/jit/loopcloning.h b/src/coreclr/jit/loopcloning.h index 9279bc783a9ec..2333d491764cd 100644 --- a/src/coreclr/jit/loopcloning.h +++ b/src/coreclr/jit/loopcloning.h @@ -135,10 +135,7 @@ exception occurs. Preconditions - 1. Loop detection has completed and the loop table is populated. - - 2. The loops that will be considered are the ones with the LPFLG_ITER flag: - "for ( ; test_condition(); i++)" + Loop detection has completed and the Compiler::m_loops is populated. Limitations @@ -147,11 +144,7 @@ exception occurs. is "hard".) There are a few other EH-related edge conditions that also cause us to reject cloning. - 2. If the loop contains RETURN blocks, and cloning those would push us over the maximum - number of allowed RETURN blocks in the function (either due to GC info encoding limitations - or otherwise), we reject cloning. - - 3. Loop increment must be `i += 1` + 2. Loop increment must be `i += 1` 4. Loop test must be `i < x` or `i <= x` where `x` is a constant, a variable, or `a.Length` for array `a` @@ -168,10 +161,6 @@ exception occurs. For non-constant (or not found) iterator variable `i` initialization, we add a dynamic check that `i >= 0`. Constant initializations can be checked statically. - 9. The cloned loop (the slow path) is not added to the loop table, meaning certain - downstream optimization passes do not see them. See - https://github.com/dotnet/runtime/issues/43713. - Assumptions 1. The assumption is that the optimization candidates collected during the @@ -181,7 +170,7 @@ exception occurs. collect additional information at the same time as identifying the optimization candidates. This later helps us to perform the optimizations during actual cloning. - 2. All loop cloning choice conditions will automatically be "AND"-ed. These are bitwise AND operations. + 2. All loop cloning choice conditions will automatically be "AND"-ed. 3. Perform short circuit AND for (array != null) side effect check before hoisting (limit <= a.length) check. diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 1bf3288fa7c3c..33428d5fd5a11 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -13220,24 +13220,14 @@ Compiler::FoldResult Compiler::fgFoldConditional(BasicBlock* block) if (cond->AsIntCon()->gtIconVal != 0) { - /* JTRUE 1 - transform the basic block into a BBJ_ALWAYS */ + // JTRUE 1 - transform the basic block into a BBJ_ALWAYS bTaken = block->GetTrueTarget(); bNotTaken = block->GetFalseTarget(); block->SetKind(BBJ_ALWAYS); } else { - /* Unmark the loop if we are removing a backwards branch */ - /* dest block must also be marked as a loop head and */ - /* We must be able to reach the backedge block */ - if (optLoopTableValid && block->GetTrueTarget()->isLoopHead() && - (block->GetTrueTarget()->bbNum <= block->bbNum) && - m_reachabilitySets->CanReach(block->GetTrueTarget(), block)) - { - optUnmarkLoopBlocks(block->GetTrueTarget(), block); - } - - /* JTRUE 0 - transform the basic block into a BBJ_ALWAYS */ + // JTRUE 0 - transform the basic block into a BBJ_ALWAYS bTaken = block->GetFalseTarget(); bNotTaken = block->GetTrueTarget(); block->SetKindAndTarget(BBJ_ALWAYS, bTaken); @@ -13336,56 +13326,6 @@ Compiler::FoldResult Compiler::fgFoldConditional(BasicBlock* block) printf("\n"); } #endif - - // Handle updates to the loop table. - // Note this is distinct from the check for BBF_LOOP_HEAD above. - // - if (optLoopTableValid) - { - for (unsigned loopNum = 0; loopNum < optLoopCount; loopNum++) - { - LoopDsc& loop = optLoopTable[loopNum]; - - // Some loops may have been already removed by - // loop unrolling or conditional folding - // - if (loop.lpIsRemoved()) - { - continue; - } - - // Removed edge from bottom -> entry? - // - if ((loop.lpBottom == block) && (loop.lpEntry == bNotTaken)) - { - // This either destroyed the loop or lessened its extent. - // We currently ignore the latter. - // - if (loop.lpEntry->countOfInEdges() == 1) - { - // We removed the only backedge. - // - JITDUMP("Removing loop " FMT_LP " (from " FMT_BB " to " FMT_BB - ") -- no longer has a backedge\n\n", - loopNum, loop.lpTop->bbNum, loop.lpBottom->bbNum); - - optMarkLoopRemoved(loopNum); - } - } - - // Removed edge from head -> entry? - // - if ((loop.lpHead == block) && (loop.lpEntry == bNotTaken)) - { - // Loop is no longer reachable from outside - // - JITDUMP("Removing loop " FMT_LP " (from " FMT_BB " to " FMT_BB ") -- no longer reachable\n\n", - loopNum, loop.lpTop->bbNum, loop.lpBottom->bbNum); - - optMarkLoopRemoved(loopNum); - } - } - } } } else if (block->KindIs(BBJ_SWITCH)) diff --git a/src/coreclr/jit/optimizebools.cpp b/src/coreclr/jit/optimizebools.cpp index b96c4ae024e40..7cca29bc7dbfe 100644 --- a/src/coreclr/jit/optimizebools.cpp +++ b/src/coreclr/jit/optimizebools.cpp @@ -1188,7 +1188,7 @@ bool OptBoolsDsc::optOptimizeBoolsChkTypeCostCond() //----------------------------------------------------------------------------- // optOptimizeBoolsUpdateTrees: Fold the trees based on fold type and comparison type, -// update the edges, unlink removed blocks and update loop table +// update the edges, and unlink removed blocks // void OptBoolsDsc::optOptimizeBoolsUpdateTrees() { @@ -1340,16 +1340,6 @@ void OptBoolsDsc::optOptimizeBoolsUpdateTrees() m_comp->ehUpdateForDeletedBlock(m_b3); } - // Update loop table - if (m_comp->optLoopTableValid) - { - m_comp->fgUpdateLoopsAfterCompacting(m_b1, m_b2); - if (optReturnBlock) - { - m_comp->fgUpdateLoopsAfterCompacting(m_b1, m_b3); - } - } - // Update IL range of first block m_b1->bbCodeOffsEnd = optReturnBlock ? m_b3->bbCodeOffsEnd : m_b2->bbCodeOffsEnd; } diff --git a/src/coreclr/jit/optimizer.cpp b/src/coreclr/jit/optimizer.cpp index 76c3662e9a799..8e5124745e9e7 100644 --- a/src/coreclr/jit/optimizer.cpp +++ b/src/coreclr/jit/optimizer.cpp @@ -21,12 +21,8 @@ void Compiler::optInit() { fgHasLoops = false; - /* Initialize the # of tracked loops to 0 */ - optLoopCount = 0; - optLoopTable = nullptr; - optLoopTableValid = false; optLoopsRequirePreHeaders = false; - optCurLoopEpoch = 0; + optNumNaturalLoopsFound = 0; #ifdef DEBUG loopAlignCandidates = 0; @@ -290,1822 +286,278 @@ void Compiler::optScaleLoopBlocks(BasicBlock* begBlk, BasicBlock* endBlk) } } -//------------------------------------------------------------------------ -// optUnmarkLoopBlocks: Unmark the blocks between 'begBlk' and 'endBlk' as part of a loop. +//---------------------------------------------------------------------------------- +// optIsLoopIncrTree: Check if loop is a tree of form v = v op const. // // Arguments: -// begBlk - first block of range. Must be marked as a loop head (BBF_LOOP_HEAD). -// endBlk - last block of range (inclusive). Must be reachable from `begBlk`. +// incr - The incr tree to be checked. // -// Operation: -// A set of blocks that were previously marked as a loop are now to be unmarked, since we have decided that -// for some reason this loop no longer exists. Basically we are just resetting the blocks bbWeight to their -// previous values. +// Return Value: +// iterVar local num if the iterVar is found, otherwise BAD_VAR_NUM. // -void Compiler::optUnmarkLoopBlocks(BasicBlock* begBlk, BasicBlock* endBlk) +unsigned Compiler::optIsLoopIncrTree(GenTree* incr) { - noway_assert(begBlk->bbNum <= endBlk->bbNum); - noway_assert(begBlk->isLoopHead()); - noway_assert(!opts.MinOpts()); - - unsigned backEdgeCount = 0; - - for (BasicBlock* const predBlock : begBlk->PredBlocks()) + GenTree* incrVal; + genTreeOps updateOper; + unsigned iterVar = incr->IsLclVarUpdateTree(&incrVal, &updateOper); + if (iterVar != BAD_VAR_NUM) { - // Is this a backward edge? (from predBlock to begBlk) - if (begBlk->bbNum > predBlock->bbNum) + // We have v = v op y type node. + switch (updateOper) { - continue; + case GT_ADD: + case GT_SUB: + case GT_MUL: + case GT_RSH: + case GT_LSH: + break; + default: + return BAD_VAR_NUM; } - // We only consider back-edges that are BBJ_COND or BBJ_ALWAYS for loops. - if (!predBlock->KindIs(BBJ_COND, BBJ_ALWAYS)) + // Increment should be by a const int. + // TODO-CQ: CLONE: allow variable increments. + if ((incrVal->gtOper != GT_CNS_INT) || (incrVal->TypeGet() != TYP_INT)) { - continue; + return BAD_VAR_NUM; } - - backEdgeCount++; } - // Only unmark the loop blocks if we have exactly one loop back edge. - if (backEdgeCount != 1) - { -#ifdef DEBUG - if (verbose) - { - if (backEdgeCount > 0) - { - printf("\nNot removing loop at " FMT_BB ", due to an additional back edge", begBlk->bbNum); - } - else if (backEdgeCount == 0) - { - printf("\nNot removing loop at " FMT_BB ", due to no back edge", begBlk->bbNum); - } - } -#endif - return; - } - noway_assert(m_reachabilitySets->CanReach(begBlk, endBlk)); + return iterVar; +} -#ifdef DEBUG - if (verbose) +//---------------------------------------------------------------------------------- +// optIsLoopTestEvalIntoTemp: +// Pattern match if the test tree is computed into a tmp +// and the "tmp" is used as jump condition for loop termination. +// +// Arguments: +// testStmt - is the JTRUE statement that is of the form: jmpTrue (Vtmp != 0) +// where Vtmp contains the actual loop test result. +// newTestStmt - contains the statement that is the actual test stmt involving +// the loop iterator. +// +// Return Value: +// Returns true if a new test tree can be obtained. +// +// Operation: +// Scan if the current stmt is a jtrue with (Vtmp != 0) as condition +// Then returns the rhs for def of Vtmp as the "test" node. +// +// Note: +// This method just retrieves what it thinks is the "test" node, +// the callers are expected to verify that "iterVar" is used in the test. +// +bool Compiler::optIsLoopTestEvalIntoTemp(Statement* testStmt, Statement** newTestStmt) +{ + GenTree* test = testStmt->GetRootNode(); + + if (test->gtOper != GT_JTRUE) { - printf("\nUnmarking a loop from " FMT_BB " to " FMT_BB, begBlk->bbNum, endBlk->bbNum); + return false; } -#endif - for (BasicBlock* const curBlk : BasicBlockRangeList(begBlk, endBlk)) - { - // Stop if we go past the last block in the loop, as it may have been deleted. - if (curBlk->bbNum > endBlk->bbNum) - { - break; - } + GenTree* relop = test->gtGetOp1(); + noway_assert(relop->OperIsCompare()); - // Don't change the block weight if it's known to be rarely run. - if (curBlk->isRunRarely()) - { - continue; - } + GenTree* opr1 = relop->AsOp()->gtOp1; + GenTree* opr2 = relop->AsOp()->gtOp2; - // Don't change the block weight if it came from profile data. - if (curBlk->hasProfileWeight()) + // Make sure we have jtrue (vtmp != 0) + if ((relop->OperGet() == GT_NE) && (opr1->OperGet() == GT_LCL_VAR) && (opr2->OperGet() == GT_CNS_INT) && + opr2->IsIntegralConst(0)) + { + // Get the previous statement to get the def (rhs) of Vtmp to see + // if the "test" is evaluated into Vtmp. + Statement* prevStmt = testStmt->GetPrevStmt(); + if (prevStmt == nullptr) { - continue; + return false; } - // Don't unmark blocks that are maximum weight. - if (curBlk->isMaxBBWeight()) + GenTree* tree = prevStmt->GetRootNode(); + if (tree->OperIs(GT_STORE_LCL_VAR) && (tree->AsLclVar()->GetLclNum() == opr1->AsLclVar()->GetLclNum()) && + tree->AsLclVar()->Data()->OperIsCompare()) { - continue; + *newTestStmt = prevStmt; + return true; } + } + return false; +} - // For curBlk to be part of a loop that starts at begBlk, curBlk must be reachable from begBlk and - // (since this is a loop) begBlk must likewise be reachable from curBlk. - // - if (m_reachabilitySets->CanReach(curBlk, begBlk) && m_reachabilitySets->CanReach(begBlk, curBlk)) - { - weight_t scale = 1.0 / BB_LOOP_WEIGHT_SCALE; - - if (!m_domTree->Dominates(curBlk, endBlk)) - { - scale *= 2; - } +//---------------------------------------------------------------------------------- +// optExtractInitTestIncr: +// Extract the "init", "test" and "incr" nodes of the loop. +// +// Arguments: +// pInitBlock - [IN/OUT] *pInitBlock is the loop head block on entry, and is set to the initBlock on exit, +// if `**ppInit` is non-null. +// bottom - Loop bottom block +// top - Loop top block +// ppInit - The init stmt of the loop if found. +// ppTest - The test stmt of the loop if found. +// ppIncr - The incr stmt of the loop if found. +// +// Return Value: +// The results are put in "ppInit", "ppTest" and "ppIncr" if the method +// returns true. Returns false if the information can't be extracted. +// Extracting the `init` is optional; if one is not found, *ppInit is set +// to nullptr. Return value will never be false if `init` is not found. +// +// Operation: +// Check if the "test" stmt is last stmt in the loop "bottom". Try to find the "incr" stmt. +// Check previous stmt of "test" to get the "incr" stmt. If it is not found it could be a loop of the +// below form. +// +// +-------<-----------------<-----------+ +// | | +// v | +// BBinit(head) -> BBcond(top) -> BBLoopBody(bottom) ---^ +// +// Check if the "incr" tree is present in the loop "top" node as the last stmt. +// Also check if the "test" tree is assigned to a tmp node and the tmp is used +// in the jtrue condition. +// +// Note: +// This method just retrieves what it thinks is the "test" node, +// the callers are expected to verify that "iterVar" is used in the test. +// +bool Compiler::optExtractInitTestIncr( + BasicBlock** pInitBlock, BasicBlock* bottom, BasicBlock* top, GenTree** ppInit, GenTree** ppTest, GenTree** ppIncr) +{ + assert(pInitBlock != nullptr); + assert(ppInit != nullptr); + assert(ppTest != nullptr); + assert(ppIncr != nullptr); - curBlk->scaleBBWeight(scale); + // Check if last two statements in the loop body are the increment of the iterator + // and the loop termination test. + noway_assert(bottom->bbStmtList != nullptr); + Statement* testStmt = bottom->lastStmt(); + noway_assert(testStmt != nullptr && testStmt->GetNextStmt() == nullptr); - JITDUMP("\n " FMT_BB "(wt=" FMT_WT ")", curBlk->bbNum, curBlk->getBBWeight(this)); - } + Statement* newTestStmt; + if (optIsLoopTestEvalIntoTemp(testStmt, &newTestStmt)) + { + testStmt = newTestStmt; } - JITDUMP("\n"); -} - -/***************************************************************************************************** - * - * Function called to update the loop table and bbWeight before removing a block - */ + // Check if we have the incr stmt before the test stmt, if we don't, + // check if incr is part of the loop "top". + Statement* incrStmt = testStmt->GetPrevStmt(); -void Compiler::optUpdateLoopsBeforeRemoveBlock(BasicBlock* block, bool skipUnmarkLoop) -{ - if (!optLoopTableValid) + // If we've added profile instrumentation, we may need to skip past a BB counter update. + // + if (opts.jitFlags->IsSet(JitFlags::JIT_FLAG_BBINSTR) && (incrStmt != nullptr) && + incrStmt->GetRootNode()->IsBlockProfileUpdate()) { - return; + incrStmt = incrStmt->GetPrevStmt(); } - noway_assert(!opts.MinOpts()); + if (incrStmt == nullptr || (optIsLoopIncrTree(incrStmt->GetRootNode()) == BAD_VAR_NUM)) + { + return false; + } - // If an unreachable block is a loop entry or bottom then the loop is unreachable. - // Special case: the block was the head of a loop - or pointing to a loop entry. + assert(testStmt != incrStmt); - for (unsigned loopNum = 0; loopNum < optLoopCount; loopNum++) + // Find the last statement in the loop pre-header which we expect to be the initialization of + // the loop iterator. + BasicBlock* initBlock = *pInitBlock; + Statement* phdrStmt = initBlock->firstStmt(); + if (phdrStmt == nullptr) { - LoopDsc& loop = optLoopTable[loopNum]; - - // Some loops may have been already removed by loop unrolling or conditional folding. - if (loop.lpIsRemoved()) + // When we build the loops, we canonicalize by introducing loop pre-headers for all loops. + // If we are rebuilding the loops, we would already have the pre-header block introduced + // the first time, which might be empty if no hoisting has yet occurred. In this case, look a + // little harder for the possible loop initialization statement. + if (initBlock->KindIs(BBJ_ALWAYS) && initBlock->TargetIs(top) && (initBlock->countOfInEdges() == 1) && + !initBlock->IsFirst() && initBlock->Prev()->bbFallsThrough()) { - continue; + initBlock = initBlock->Prev(); + phdrStmt = initBlock->firstStmt(); } + } - // Avoid printing to the JitDump unless we're actually going to change something. - // If we call reportBefore, then we're going to change the loop table, and we should print the - // `reportAfter` info as well. Only print the `reportBefore` info once, if multiple changes to - // the table are made. - INDEBUG(bool reportedBefore = false); + if (phdrStmt != nullptr) + { + Statement* initStmt = phdrStmt->GetPrevStmt(); + noway_assert(initStmt != nullptr && (initStmt->GetNextStmt() == nullptr)); - auto reportBefore = [&]() { + // If it is a duplicated loop condition, skip it. + if (initStmt->GetRootNode()->OperIs(GT_JTRUE)) + { + bool doGetPrev = true; #ifdef DEBUG - if (verbose && !reportedBefore) + if (opts.optRepeat) { - printf("optUpdateLoopsBeforeRemoveBlock " FMT_BB " Before: ", block->bbNum); - optPrintLoopInfo(loopNum); - printf("\n"); - reportedBefore = true; + // Previous optimization passes may have inserted compiler-generated + // statements other than duplicated loop conditions. + doGetPrev = (initStmt->GetPrevStmt() != nullptr); } #endif // DEBUG - }; - - auto reportAfter = [&]() { -#ifdef DEBUG - if (verbose && reportedBefore) + if (doGetPrev) { - printf("optUpdateLoopsBeforeRemoveBlock " FMT_BB " After: ", block->bbNum); - optPrintLoopInfo(loopNum); - printf("\n"); + initStmt = initStmt->GetPrevStmt(); } -#endif // DEBUG - }; - - if ((block == loop.lpEntry) || (block == loop.lpBottom) || (block == loop.lpTop)) - { - reportBefore(); - optMarkLoopRemoved(loopNum); - reportAfter(); - continue; + noway_assert(initStmt != nullptr); } - // If the loop is still in the table any block in the loop must be reachable. - - noway_assert((loop.lpEntry != block) && (loop.lpBottom != block)); + *ppInit = initStmt->GetRootNode(); + *pInitBlock = initBlock; + } + else + { + *ppInit = nullptr; + } - if (loop.lpExit == block) - { - reportBefore(); - assert(loop.lpExitCnt == 1); - --loop.lpExitCnt; - loop.lpExit = nullptr; - } + *ppTest = testStmt->GetRootNode(); + *ppIncr = incrStmt->GetRootNode(); - // If `block` flows to the loop entry then the whole loop will become unreachable if it is the - // only non-loop predecessor. + return true; +} - bool removeLoop = false; - if (!loop.lpContains(block)) +#ifdef DEBUG +void Compiler::optCheckPreds() +{ + for (BasicBlock* const block : Blocks()) + { + for (BasicBlock* const predBlock : block->PredBlocks()) { - for (BasicBlock* const succ : block->Succs()) + // make sure this pred is part of the BB list + BasicBlock* bb; + for (bb = fgFirstBB; bb; bb = bb->Next()) { - if (loop.lpEntry == succ) + if (bb == predBlock) { - removeLoop = true; break; } } - - if (removeLoop) + noway_assert(bb); + switch (bb->GetKind()) { - // If the entry has any non-loop block that is not the known 'block' predecessor of entry - // (found above), then don't remove the loop. - for (BasicBlock* const predBlock : loop.lpEntry->PredBlocks()) - { - if (!loop.lpContains(predBlock) && (predBlock != block)) + case BBJ_COND: + if (bb->TrueTargetIs(block)) { - removeLoop = false; break; } - } - } - } - - if (removeLoop) - { - reportBefore(); - optMarkLoopRemoved(loopNum); - } - else if (loop.lpHead == block) - { - reportBefore(); - /* The loop has a new head - Just update the loop table */ - loop.lpHead = block->Prev(); - } - - reportAfter(); - } - - if (!skipUnmarkLoop && // If we want to unmark this loop... - (fgCurBBEpochSize == fgBBNumMax + 1) && // We didn't add new blocks since last renumber... - (m_reachabilitySets != nullptr) && // Given the reachability sets are computed and valid... - (fgCurBBEpochSize == fgDomBBcount + 1)) - { - // This block must reach conditionally or always - - if (block->KindIs(BBJ_ALWAYS) && // This block always reaches - block->GetTarget()->isLoopHead() && // to a loop head... - (block->GetTarget()->bbNum <= block->bbNum) && // This is a backedge... - m_reachabilitySets->CanReach(block->GetTarget(), block)) // Block's back edge target can reach block... - { - optUnmarkLoopBlocks(block->GetTarget(), block); // Unscale the blocks in such loop. - } - else if (block->KindIs(BBJ_COND) && // This block conditionally reaches - block->GetTrueTarget()->isLoopHead() && // to a loop head... - (block->GetTrueTarget()->bbNum <= block->bbNum) && // This is a backedge... - m_reachabilitySets->CanReach(block->GetTrueTarget(), block)) // Block's back edge target can reach - // block... - { - optUnmarkLoopBlocks(block->GetTrueTarget(), block); // Unscale the blocks in such loop. - } - } -} - -#ifdef DEBUG - -/***************************************************************************** - * - * Print loop info in an uniform way. - */ - -void Compiler::optPrintLoopInfo(const LoopDsc* loop, bool printVerbose /* = false */) -{ - assert(optLoopTable != nullptr); - assert((&optLoopTable[0] <= loop) && (loop < &optLoopTable[optLoopCount])); - - unsigned lnum = (unsigned)(loop - optLoopTable); - assert(lnum < optLoopCount); - assert(&optLoopTable[lnum] == loop); - - if (loop->lpIsRemoved()) - { - // If a loop has been removed, it might be dangerous to print its fields (e.g., loop unrolling - // nulls out the lpHead field). - printf(FMT_LP " REMOVED", lnum); - return; - } - - printf(FMT_LP ", from " FMT_BB " to " FMT_BB " (Head=" FMT_BB ", Entry=" FMT_BB, lnum, loop->lpTop->bbNum, - loop->lpBottom->bbNum, loop->lpHead->bbNum, loop->lpEntry->bbNum); - - if (loop->lpExitCnt == 1) - { - printf(", Exit=" FMT_BB, loop->lpExit->bbNum); - } - else - { - printf(", ExitCnt=%d", loop->lpExitCnt); - } - - if (loop->lpParent != BasicBlock::NOT_IN_LOOP) - { - printf(", parent=" FMT_LP, loop->lpParent); - } - printf(")"); - - if (printVerbose) - { - if (loop->lpChild != BasicBlock::NOT_IN_LOOP) - { - printf(", child loop = " FMT_LP, loop->lpChild); - } - if (loop->lpSibling != BasicBlock::NOT_IN_LOOP) - { - printf(", sibling loop = " FMT_LP, loop->lpSibling); - } - - // If an iterator loop print the iterator and the initialization. - if (loop->lpFlags & LPFLG_ITER) - { - printf(" [over V%02u", loop->lpIterVar()); - printf(" ("); - printf(GenTree::OpName(loop->lpIterOper())); - printf(" %d)", loop->lpIterConst()); - - if (loop->lpFlags & LPFLG_CONST_INIT) - { - printf(" from %d", loop->lpConstInit); - } - - if (loop->lpFlags & LPFLG_CONST_INIT) - { - if (loop->lpInitBlock != loop->lpHead) - { - printf(" (in " FMT_BB ")", loop->lpInitBlock->bbNum); - } - } - - // If a simple test condition print operator and the limits */ - printf(" %s", GenTree::OpName(loop->lpTestOper())); - - if (loop->lpFlags & LPFLG_CONST_LIMIT) - { - printf(" %d", loop->lpConstLimit()); - if (loop->lpFlags & LPFLG_SIMD_LIMIT) - { - printf(" (simd)"); - } - } - if (loop->lpFlags & LPFLG_VAR_LIMIT) - { - printf(" V%02u", loop->lpVarLimit()); - } - if (loop->lpFlags & LPFLG_ARRLEN_LIMIT) - { - ArrIndex* index = new (getAllocator(CMK_DebugOnly)) ArrIndex(getAllocator(CMK_DebugOnly)); - if (loop->lpArrLenLimit(this, index)) - { - printf(" "); - index->Print(); - printf(".Length"); - } - else - { - printf(" ???.Length"); - } - } - - printf("]"); - } - - // Print the flags - - if (loop->lpFlags & LPFLG_CONTAINS_CALL) - { - printf(" call"); - } - if (loop->lpFlags & LPFLG_HAS_PREHEAD) - { - printf(" prehead"); - } - if (loop->lpFlags & LPFLG_DONT_UNROLL) - { - printf(" !unroll"); - } - if (loop->lpFlags & LPFLG_ASGVARS_YES) - { - printf(" avyes"); - } - if (loop->lpFlags & LPFLG_ASGVARS_INC) - { - printf(" avinc"); - } - } -} - -void Compiler::optPrintLoopInfo(unsigned lnum, bool printVerbose /* = false */) -{ - assert(lnum < optLoopCount); - - const LoopDsc& loop = optLoopTable[lnum]; - optPrintLoopInfo(&loop, printVerbose); -} - -//------------------------------------------------------------------------ -// optPrintLoopTable: Print the loop table -// -void Compiler::optPrintLoopTable() -{ - printf("\n*************** Natural loop table\n"); - - if (optLoopCount == 0) - { - printf("No loops\n"); - } - else - { - for (unsigned loopInd = 0; loopInd < optLoopCount; loopInd++) - { - optPrintLoopInfo(loopInd, /* verbose */ true); - printf("\n"); - } - } - - printf("\n"); -} - -#endif // DEBUG - -//------------------------------------------------------------------------ -// optPopulateInitInfo: Populate loop init info in the loop table. -// We assume the iteration variable is initialized already and check appropriately. -// This only checks for the special case of a constant initialization. -// -// Arguments: -// loopInd - loop index -// initBlock - block in which the initialization lives. -// init - the tree that is supposed to initialize the loop iterator. Might be nullptr. -// iterVar - loop iteration variable. -// -// Return Value: -// "true" if a constant initializer was found. -// -// Operation: -// The 'init' tree is checked if its lhs is a local and rhs is a const. -// -bool Compiler::optPopulateInitInfo(unsigned loopInd, BasicBlock* initBlock, GenTree* init, unsigned iterVar) -{ - // Operator should be STORE_LCL_VAR - if ((init == nullptr) || !init->OperIs(GT_STORE_LCL_VAR) || (init->AsLclVar()->GetLclNum() != iterVar)) - { - return false; - } - - // Value must be constant. TODO-CQ: CLONE: Add arr length for descending loops. - GenTree* initValue = init->AsLclVar()->Data(); - if (!initValue->IsCnsIntOrI() || (initValue->TypeGet() != TYP_INT)) - { - return false; - } - - // We found an initializer in the `initBlock` block. For this to be used, we need to make sure the - // "iterVar" initialization is never skipped. That is, every pred of ENTRY other than HEAD is in the loop. - // We allow one special case: the HEAD block is an empty predecessor to ENTRY, and the initBlock is the - // only predecessor to HEAD. This handles the case where we rebuild the loop table (after inserting - // pre-headers) and we still want to find the initializer before the pre-header block. - for (BasicBlock* const predBlock : optLoopTable[loopInd].lpEntry->PredBlocks()) - { - if (!optLoopTable[loopInd].lpContains(predBlock)) - { - bool initBlockOk = (predBlock == initBlock); - if (!initBlockOk) - { - if (predBlock->KindIs(BBJ_ALWAYS) && predBlock->TargetIs(optLoopTable[loopInd].lpEntry) && - (predBlock->countOfInEdges() == 1) && (predBlock->firstStmt() == nullptr) && - !predBlock->IsFirst() && predBlock->Prev()->bbFallsThrough()) - { - initBlockOk = true; - } - } - if (!initBlockOk) - { - JITDUMP(FMT_LP ": initialization not guaranteed from " FMT_BB " through to entry block " FMT_BB - " from pred " FMT_BB "; ignore constant initializer\n", - loopInd, initBlock->bbNum, optLoopTable[loopInd].lpEntry->bbNum, predBlock->bbNum); - return false; - } - } - } - - optLoopTable[loopInd].lpFlags |= LPFLG_CONST_INIT; - optLoopTable[loopInd].lpConstInit = (int)initValue->AsIntCon()->gtIconVal; - optLoopTable[loopInd].lpInitBlock = initBlock; - - return true; -} - -//---------------------------------------------------------------------------------- -// optCheckIterInLoopTest: Check if iteration variable is used in loop test. -// -// Arguments: -// loopInd - loop index -// test - "jtrue" tree or a store of the loop iteration termination condition -// iterVar - loop iteration variable. -// -// Operation: -// The test tree is parsed to check if "iterVar" matches the lhs of the condition -// and the rhs limit is extracted from the "test" tree. The limit information is -// added to the loop table. -// -// Return Value: -// "false" if the loop table could not be populated with the loop test info or -// if the test condition doesn't involve iterVar. -// -bool Compiler::optCheckIterInLoopTest(unsigned loopInd, GenTree* test, unsigned iterVar) -{ - // Obtain the relop from the "test" tree. - GenTree* relop; - if (test->OperIs(GT_JTRUE)) - { - relop = test->gtGetOp1(); - } - else - { - assert(test->OperIs(GT_STORE_LCL_VAR)); - relop = test->AsLclVar()->Data(); - } - - noway_assert(relop->OperIsCompare()); - - GenTree* opr1 = relop->AsOp()->gtOp1; - GenTree* opr2 = relop->AsOp()->gtOp2; - - GenTree* iterOp; - GenTree* limitOp; - - // Make sure op1 or op2 is the iterVar. - if (opr1->gtOper == GT_LCL_VAR && opr1->AsLclVarCommon()->GetLclNum() == iterVar) - { - iterOp = opr1; - limitOp = opr2; - } - else if (opr2->gtOper == GT_LCL_VAR && opr2->AsLclVarCommon()->GetLclNum() == iterVar) - { - iterOp = opr2; - limitOp = opr1; - } - else - { - return false; - } - - if (iterOp->gtType != TYP_INT) - { - return false; - } - - // Mark the iterator node. - iterOp->gtFlags |= GTF_VAR_ITERATOR; - - // Check what type of limit we have - constant, variable or arr-len. - if (limitOp->gtOper == GT_CNS_INT) - { - optLoopTable[loopInd].lpFlags |= LPFLG_CONST_LIMIT; - if ((limitOp->gtFlags & GTF_ICON_SIMD_COUNT) != 0) - { - optLoopTable[loopInd].lpFlags |= LPFLG_SIMD_LIMIT; - } - } - else if (limitOp->gtOper == GT_LCL_VAR) - { - // See if limit var is a loop invariant - // - if (!optIsVarAssgLoop(loopInd, limitOp->AsLclVarCommon()->GetLclNum())) - { - optLoopTable[loopInd].lpFlags |= LPFLG_VAR_LIMIT; - } - else - { - JITDUMP("Limit var V%02u modifiable in " FMT_LP "\n", limitOp->AsLclVarCommon()->GetLclNum(), loopInd); - } - } - else if (limitOp->gtOper == GT_ARR_LENGTH) - { - // See if limit array is a loop invariant - // - GenTree* const array = limitOp->AsArrLen()->ArrRef(); - - if (array->OperIs(GT_LCL_VAR)) - { - if (!optIsVarAssgLoop(loopInd, array->AsLclVarCommon()->GetLclNum())) - { - optLoopTable[loopInd].lpFlags |= LPFLG_ARRLEN_LIMIT; - } - else - { - JITDUMP("Array limit var V%02u modifiable in " FMT_LP "\n", array->AsLclVarCommon()->GetLclNum(), - loopInd); + noway_assert(bb->FalseTargetIs(block)); + break; + case BBJ_EHFILTERRET: + case BBJ_ALWAYS: + case BBJ_EHCATCHRET: + noway_assert(bb->TargetIs(block)); + break; + default: + break; } } - else - { - JITDUMP("Array limit tree [%06u] not analyzable in " FMT_LP "\n", dspTreeID(limitOp), loopInd); - } - } - else - { - JITDUMP("Loop limit tree [%06u] not analyzable in " FMT_LP "\n", dspTreeID(limitOp), loopInd); } - - // Were we able to successfully analyze the limit? - // - const bool analyzedLimit = - (optLoopTable[loopInd].lpFlags & (LPFLG_CONST_LIMIT | LPFLG_VAR_LIMIT | LPFLG_ARRLEN_LIMIT)) != 0; - - // Save the type of the comparison between the iterator and the limit. - // - optLoopTable[loopInd].lpTestTree = relop; - - return analyzedLimit; -} - -//---------------------------------------------------------------------------------- -// optIsLoopIncrTree: Check if loop is a tree of form v = v op const. -// -// Arguments: -// incr - The incr tree to be checked. -// -// Return Value: -// iterVar local num if the iterVar is found, otherwise BAD_VAR_NUM. -// -unsigned Compiler::optIsLoopIncrTree(GenTree* incr) -{ - GenTree* incrVal; - genTreeOps updateOper; - unsigned iterVar = incr->IsLclVarUpdateTree(&incrVal, &updateOper); - if (iterVar != BAD_VAR_NUM) - { - // We have v = v op y type node. - switch (updateOper) - { - case GT_ADD: - case GT_SUB: - case GT_MUL: - case GT_RSH: - case GT_LSH: - break; - default: - return BAD_VAR_NUM; - } - - // Increment should be by a const int. - // TODO-CQ: CLONE: allow variable increments. - if ((incrVal->gtOper != GT_CNS_INT) || (incrVal->TypeGet() != TYP_INT)) - { - return BAD_VAR_NUM; - } - } - - return iterVar; -} - -//---------------------------------------------------------------------------------- -// optComputeIterInfo: Check tree is loop increment of a lcl that is loop-invariant. -// -// Arguments: -// incr - tree that increments the loop iterator. v+=1 or v=v+1. -// from, to - range of blocks that comprise the loop body -// pIterVar - see return value. -// -// Return Value: -// Returns true if iterVar "v" can be returned in "pIterVar", otherwise returns -// false. -// -// Operation: -// Check if the "incr" tree is a "v=v+1 or v+=1" type tree and make sure it is not -// otherwise modified in the loop. -// -bool Compiler::optComputeIterInfo(GenTree* incr, BasicBlock* from, BasicBlock* to, unsigned* pIterVar) -{ - const unsigned iterVar = optIsLoopIncrTree(incr); - - if (iterVar == BAD_VAR_NUM) - { - return false; - } - - // Note we can't use optIsVarAssgLoop here, as iterVar is indeed - // assigned within the loop. - // - // Bail on promoted case, otherwise we'd have to search the loop - // for both iterVar and its parent. - // - // Bail on the potentially aliased case. - // - LclVarDsc* const iterVarDsc = lvaGetDesc(iterVar); - - if (iterVarDsc->lvIsStructField) - { - JITDUMP("iterVar V%02u is a promoted field\n", iterVar); - return false; - } - - if (iterVarDsc->IsAddressExposed()) - { - JITDUMP("iterVar V%02u is address exposed\n", iterVar); - return false; - } - - if (optIsVarAssigned(from, to, incr, iterVar)) - { - JITDUMP("iterVar V%02u is assigned in loop\n", iterVar); - return false; - } - - JITDUMP("iterVar V%02u is invariant in loop (with the exception of the update in [%06u])\n", iterVar, - dspTreeID(incr)); - - *pIterVar = iterVar; - return true; -} - -//---------------------------------------------------------------------------------- -// optIsLoopTestEvalIntoTemp: -// Pattern match if the test tree is computed into a tmp -// and the "tmp" is used as jump condition for loop termination. -// -// Arguments: -// testStmt - is the JTRUE statement that is of the form: jmpTrue (Vtmp != 0) -// where Vtmp contains the actual loop test result. -// newTestStmt - contains the statement that is the actual test stmt involving -// the loop iterator. -// -// Return Value: -// Returns true if a new test tree can be obtained. -// -// Operation: -// Scan if the current stmt is a jtrue with (Vtmp != 0) as condition -// Then returns the rhs for def of Vtmp as the "test" node. -// -// Note: -// This method just retrieves what it thinks is the "test" node, -// the callers are expected to verify that "iterVar" is used in the test. -// -bool Compiler::optIsLoopTestEvalIntoTemp(Statement* testStmt, Statement** newTestStmt) -{ - GenTree* test = testStmt->GetRootNode(); - - if (test->gtOper != GT_JTRUE) - { - return false; - } - - GenTree* relop = test->gtGetOp1(); - noway_assert(relop->OperIsCompare()); - - GenTree* opr1 = relop->AsOp()->gtOp1; - GenTree* opr2 = relop->AsOp()->gtOp2; - - // Make sure we have jtrue (vtmp != 0) - if ((relop->OperGet() == GT_NE) && (opr1->OperGet() == GT_LCL_VAR) && (opr2->OperGet() == GT_CNS_INT) && - opr2->IsIntegralConst(0)) - { - // Get the previous statement to get the def (rhs) of Vtmp to see - // if the "test" is evaluated into Vtmp. - Statement* prevStmt = testStmt->GetPrevStmt(); - if (prevStmt == nullptr) - { - return false; - } - - GenTree* tree = prevStmt->GetRootNode(); - if (tree->OperIs(GT_STORE_LCL_VAR) && (tree->AsLclVar()->GetLclNum() == opr1->AsLclVar()->GetLclNum()) && - tree->AsLclVar()->Data()->OperIsCompare()) - { - *newTestStmt = prevStmt; - return true; - } - } - return false; -} - -//---------------------------------------------------------------------------------- -// optExtractInitTestIncr: -// Extract the "init", "test" and "incr" nodes of the loop. -// -// Arguments: -// pInitBlock - [IN/OUT] *pInitBlock is the loop head block on entry, and is set to the initBlock on exit, -// if `**ppInit` is non-null. -// bottom - Loop bottom block -// top - Loop top block -// ppInit - The init stmt of the loop if found. -// ppTest - The test stmt of the loop if found. -// ppIncr - The incr stmt of the loop if found. -// -// Return Value: -// The results are put in "ppInit", "ppTest" and "ppIncr" if the method -// returns true. Returns false if the information can't be extracted. -// Extracting the `init` is optional; if one is not found, *ppInit is set -// to nullptr. Return value will never be false if `init` is not found. -// -// Operation: -// Check if the "test" stmt is last stmt in the loop "bottom". Try to find the "incr" stmt. -// Check previous stmt of "test" to get the "incr" stmt. If it is not found it could be a loop of the -// below form. -// -// +-------<-----------------<-----------+ -// | | -// v | -// BBinit(head) -> BBcond(top) -> BBLoopBody(bottom) ---^ -// -// Check if the "incr" tree is present in the loop "top" node as the last stmt. -// Also check if the "test" tree is assigned to a tmp node and the tmp is used -// in the jtrue condition. -// -// Note: -// This method just retrieves what it thinks is the "test" node, -// the callers are expected to verify that "iterVar" is used in the test. -// -bool Compiler::optExtractInitTestIncr( - BasicBlock** pInitBlock, BasicBlock* bottom, BasicBlock* top, GenTree** ppInit, GenTree** ppTest, GenTree** ppIncr) -{ - assert(pInitBlock != nullptr); - assert(ppInit != nullptr); - assert(ppTest != nullptr); - assert(ppIncr != nullptr); - - // Check if last two statements in the loop body are the increment of the iterator - // and the loop termination test. - noway_assert(bottom->bbStmtList != nullptr); - Statement* testStmt = bottom->lastStmt(); - noway_assert(testStmt != nullptr && testStmt->GetNextStmt() == nullptr); - - Statement* newTestStmt; - if (optIsLoopTestEvalIntoTemp(testStmt, &newTestStmt)) - { - testStmt = newTestStmt; - } - - // Check if we have the incr stmt before the test stmt, if we don't, - // check if incr is part of the loop "top". - Statement* incrStmt = testStmt->GetPrevStmt(); - - // If we've added profile instrumentation, we may need to skip past a BB counter update. - // - if (opts.jitFlags->IsSet(JitFlags::JIT_FLAG_BBINSTR) && (incrStmt != nullptr) && - incrStmt->GetRootNode()->IsBlockProfileUpdate()) - { - incrStmt = incrStmt->GetPrevStmt(); - } - - if (incrStmt == nullptr || (optIsLoopIncrTree(incrStmt->GetRootNode()) == BAD_VAR_NUM)) - { - return false; - } - - assert(testStmt != incrStmt); - - // Find the last statement in the loop pre-header which we expect to be the initialization of - // the loop iterator. - BasicBlock* initBlock = *pInitBlock; - Statement* phdrStmt = initBlock->firstStmt(); - if (phdrStmt == nullptr) - { - // When we build the loop table, we canonicalize by introducing loop pre-headers for all loops. - // If we are rebuilding the loop table, we would already have the pre-header block introduced - // the first time, which might be empty if no hoisting has yet occurred. In this case, look a - // little harder for the possible loop initialization statement. - if (initBlock->KindIs(BBJ_ALWAYS) && initBlock->TargetIs(top) && (initBlock->countOfInEdges() == 1) && - !initBlock->IsFirst() && initBlock->Prev()->bbFallsThrough()) - { - initBlock = initBlock->Prev(); - phdrStmt = initBlock->firstStmt(); - } - } - - if (phdrStmt != nullptr) - { - Statement* initStmt = phdrStmt->GetPrevStmt(); - noway_assert(initStmt != nullptr && (initStmt->GetNextStmt() == nullptr)); - - // If it is a duplicated loop condition, skip it. - if (initStmt->GetRootNode()->OperIs(GT_JTRUE)) - { - bool doGetPrev = true; -#ifdef DEBUG - if (opts.optRepeat) - { - // Previous optimization passes may have inserted compiler-generated - // statements other than duplicated loop conditions. - doGetPrev = (initStmt->GetPrevStmt() != nullptr); - } -#endif // DEBUG - if (doGetPrev) - { - initStmt = initStmt->GetPrevStmt(); - } - noway_assert(initStmt != nullptr); - } - - *ppInit = initStmt->GetRootNode(); - *pInitBlock = initBlock; - } - else - { - *ppInit = nullptr; - } - - *ppTest = testStmt->GetRootNode(); - *ppIncr = incrStmt->GetRootNode(); - - return true; -} - -/***************************************************************************** - * - * Record the loop in the loop table. Return true if successful, false if - * out of entries in loop table. - */ - -bool Compiler::optRecordLoop( - BasicBlock* head, BasicBlock* top, BasicBlock* entry, BasicBlock* bottom, BasicBlock* exit, unsigned char exitCnt) -{ - if (exitCnt == 1) - { - noway_assert(exit != nullptr); - } - - // Record this loop in the table, if there's room. - - assert(optLoopCount <= BasicBlock::MAX_LOOP_NUM); - if (optLoopCount == BasicBlock::MAX_LOOP_NUM) - { -#if COUNT_LOOPS - loopOverflowThisMethod = true; -#endif - return false; - } - - // Assumed preconditions on the loop we're adding. - assert(top->bbNum <= entry->bbNum); - assert(entry->bbNum <= bottom->bbNum); - assert(head->bbNum < top->bbNum || head->bbNum > bottom->bbNum); - - unsigned char loopInd = optLoopCount; - - if (optLoopTable == nullptr) - { - assert(loopInd == 0); - optLoopTable = getAllocator(CMK_LoopOpt).allocate(BasicBlock::MAX_LOOP_NUM); - - NewLoopEpoch(); - } - else - { - // If the new loop contains any existing ones, add it in the right place. - for (unsigned char prevPlus1 = optLoopCount; prevPlus1 > 0; prevPlus1--) - { - unsigned char prev = prevPlus1 - 1; - if (optLoopTable[prev].lpContainedBy(top, bottom)) - { - loopInd = prev; - } - } - // Move up any loops if necessary. - for (unsigned j = optLoopCount; j > loopInd; j--) - { - optLoopTable[j] = optLoopTable[j - 1]; - } - } - -#ifdef DEBUG - for (unsigned i = loopInd + 1; i < optLoopCount; i++) - { - // The loop is well-formed. - assert(optLoopTable[i].lpWellFormed()); - // Check for disjoint. - if (optLoopTable[i].lpDisjoint(top, bottom)) - { - continue; - } - // Otherwise, assert complete containment (of optLoopTable[i] in new loop). - assert(optLoopTable[i].lpContainedBy(top, bottom)); - } -#endif // DEBUG - - bool loopInsertedAtEnd = (loopInd == optLoopCount); - optLoopCount++; - - optLoopTable[loopInd].lpHead = head; - optLoopTable[loopInd].lpTop = top; - optLoopTable[loopInd].lpBottom = bottom; - optLoopTable[loopInd].lpEntry = entry; - optLoopTable[loopInd].lpExit = exit; - optLoopTable[loopInd].lpExitCnt = exitCnt; - - optLoopTable[loopInd].lpParent = BasicBlock::NOT_IN_LOOP; - optLoopTable[loopInd].lpChild = BasicBlock::NOT_IN_LOOP; - optLoopTable[loopInd].lpSibling = BasicBlock::NOT_IN_LOOP; - - optLoopTable[loopInd].lpAsgVars = AllVarSetOps::UninitVal(); - - optLoopTable[loopInd].lpFlags = LPFLG_EMPTY; - - // We haven't yet recorded any side effects. - for (MemoryKind memoryKind : allMemoryKinds()) - { - optLoopTable[loopInd].lpLoopHasMemoryHavoc[memoryKind] = false; - } - optLoopTable[loopInd].lpFieldsModified = nullptr; - optLoopTable[loopInd].lpArrayElemTypesModified = nullptr; - - // - // Try to find loops that have an iterator (i.e. for-like loops) "for (init; test; incr){ ... }" - // We have the following restrictions: - // 1. The loop condition must be a simple one i.e. only one JTRUE node - // 2. There must be a loop iterator (a local var) that is - // incremented (decremented or lsh, rsh, mul) with a constant value - // 3. The iterator is incremented exactly once - // 4. The loop condition must use the iterator. - // 5. Finding a constant initializer is optional; if the initializer is not found, or is not constant, - // it is still considered a for-like loop. - // - if (bottom->KindIs(BBJ_COND)) - { - GenTree* init; - GenTree* test; - GenTree* incr; - BasicBlock* initBlock = head; - if (!optExtractInitTestIncr(&initBlock, bottom, top, &init, &test, &incr)) - { - JITDUMP(FMT_LP ": couldn't find init/test/incr; not LPFLG_ITER loop\n", loopInd); - goto DONE_LOOP; - } - - unsigned iterVar = BAD_VAR_NUM; - if (!optComputeIterInfo(incr, top, bottom, &iterVar)) - { - JITDUMP(FMT_LP ": increment expression not appropriate form, or not loop invariant; not LPFLG_ITER loop\n", - loopInd); - goto DONE_LOOP; - } - - optPopulateInitInfo(loopInd, initBlock, init, iterVar); - - // Check that the iterator is used in the loop condition. - if (!optCheckIterInLoopTest(loopInd, test, iterVar)) - { - JITDUMP(FMT_LP ": iterator V%02u fails analysis of loop condition [%06u]; not LPFLG_ITER loop\n", loopInd, - iterVar, dspTreeID(test)); - goto DONE_LOOP; - } - - // We know the loop has an iterator at this point; flag it as LPFLG_ITER. - JITDUMP(FMT_LP ": setting LPFLG_ITER\n", loopInd); - optLoopTable[loopInd].lpFlags |= LPFLG_ITER; - - // Record iterator. - optLoopTable[loopInd].lpIterTree = incr; - -#if COUNT_LOOPS - iterLoopCount++; - - // Check if a constant iteration loop. - if ((optLoopTable[loopInd].lpFlags & LPFLG_CONST_INIT) && (optLoopTable[loopInd].lpFlags & LPFLG_CONST_LIMIT)) - { - // This is a constant loop. - constIterLoopCount++; - } -#endif - } - -DONE_LOOP: - -#ifdef DEBUG - if (verbose) - { - printf("Recorded loop %s", loopInsertedAtEnd ? "" : "(extended) "); - optPrintLoopInfo(loopInd, /* verbose */ true); - printf("\n"); - } -#endif // DEBUG - - return true; -} - -#ifdef DEBUG -void Compiler::optCheckPreds() -{ - for (BasicBlock* const block : Blocks()) - { - for (BasicBlock* const predBlock : block->PredBlocks()) - { - // make sure this pred is part of the BB list - BasicBlock* bb; - for (bb = fgFirstBB; bb; bb = bb->Next()) - { - if (bb == predBlock) - { - break; - } - } - noway_assert(bb); - switch (bb->GetKind()) - { - case BBJ_COND: - if (bb->TrueTargetIs(block)) - { - break; - } - noway_assert(bb->FalseTargetIs(block)); - break; - case BBJ_EHFILTERRET: - case BBJ_ALWAYS: - case BBJ_EHCATCHRET: - noway_assert(bb->TargetIs(block)); - break; - default: - break; - } - } - } -} - -#endif // DEBUG - -namespace -{ -//------------------------------------------------------------------------ -// LoopSearch: Class that handles scanning a range of blocks to detect a loop, -// moving blocks to make the loop body contiguous, and recording the loop. -// -// We will use the following terminology: -// HEAD - the basic block that flows into the loop ENTRY block (Currently MUST be lexically before entry). -// Not part of the looping of the loop. -// TOP - the target of the backward edge from BOTTOM, and the lexically first basic block (in bbNext order) -// within this loop. -// BOTTOM - the lexically last block in the loop (i.e. the block from which we jump to the top) -// EXIT - the predecessor of loop's unique exit edge, if it has a unique exit edge; else nullptr -// ENTRY - the entry in the loop (not necessarily the TOP), but there must be only one entry -// -// We (currently) require the body of a loop to be a contiguous (in bbNext order) sequence of basic blocks. -// When the loop is identified, blocks will be moved out to make it a compact contiguous region if possible, -// and in cases where compaction is not possible, we'll subsequently treat all blocks in the lexical range -// between TOP and BOTTOM as part of the loop even if they aren't part of the SCC. -// Regarding nesting: Since a given block can only have one back-edge (we only detect loops with back-edges -// from BBJ_COND or BBJ_ALWAYS blocks), no two loops will share the same BOTTOM. Two loops may share the -// same TOP/ENTRY as reported by LoopSearch, and optCanonicalizeLoopNest will subsequently re-write -// the CFG so that no two loops share the same TOP/ENTRY anymore. -// -// | -// v -// head -// | -// | top <--+ -// | | | -// | ... | -// | | | -// | v | -// +---> entry | -// | | -// ... | -// | | -// v | -// +-- exit/tail | -// | | | -// | ... | -// | | | -// | v | -// | bottom ---+ -// | -// +------+ -// | -// v -// -class LoopSearch -{ - - // Keeping track of which blocks are in the loop requires two block sets since we may add blocks - // as we go but the BlockSet type's max ID doesn't increase to accommodate them. Define a helper - // struct to make the ensuing code more readable. - struct LoopBlockSet - { - private: - // Keep track of blocks with bbNum <= oldBlockMaxNum in a regular BlockSet, since - // it can hold all of them. - BlockSet oldBlocksInLoop; // Blocks with bbNum <= oldBlockMaxNum - - // Keep track of blocks with bbNum > oldBlockMaxNum in a separate BlockSet, but - // indexing them by (blockNum - oldBlockMaxNum); since we won't generate more than - // one new block per old block, this must be sufficient to track any new blocks. - BlockSet newBlocksInLoop; // Blocks with bbNum > oldBlockMaxNum - - Compiler* comp; - unsigned int oldBlockMaxNum; - - public: - LoopBlockSet(Compiler* comp) - : oldBlocksInLoop(BlockSetOps::UninitVal()) - , newBlocksInLoop(BlockSetOps::UninitVal()) - , comp(comp) - , oldBlockMaxNum(comp->fgBBNumMax) - { - } - - void Reset(unsigned int seedBlockNum) - { - if (BlockSetOps::MayBeUninit(oldBlocksInLoop)) - { - // Either the block sets are uninitialized (and long), so we need to initialize - // them (and allocate their backing storage), or they are short and empty, so - // assigning MakeEmpty to them is as cheap as ClearD. - oldBlocksInLoop = BlockSetOps::MakeEmpty(comp); - newBlocksInLoop = BlockSetOps::MakeEmpty(comp); - } - else - { - // We know the backing storage is already allocated, so just clear it. - BlockSetOps::ClearD(comp, oldBlocksInLoop); - BlockSetOps::ClearD(comp, newBlocksInLoop); - } - assert(seedBlockNum <= oldBlockMaxNum); - BlockSetOps::AddElemD(comp, oldBlocksInLoop, seedBlockNum); - } - - bool CanRepresent(unsigned int blockNum) - { - // We can represent old blocks up to oldBlockMaxNum, and - // new blocks up to 2 * oldBlockMaxNum. - return (blockNum <= 2 * oldBlockMaxNum); - } - - bool IsMember(unsigned int blockNum) - { - if (blockNum > oldBlockMaxNum) - { - return BlockSetOps::IsMember(comp, newBlocksInLoop, blockNum - oldBlockMaxNum); - } - else - { - return BlockSetOps::IsMember(comp, oldBlocksInLoop, blockNum); - } - } - - void Insert(unsigned int blockNum) - { - if (blockNum > oldBlockMaxNum) - { - BlockSetOps::AddElemD(comp, newBlocksInLoop, blockNum - oldBlockMaxNum); - } - else - { - BlockSetOps::AddElemD(comp, oldBlocksInLoop, blockNum); - } - } - - bool TestAndInsert(unsigned int blockNum) - { - if (blockNum > oldBlockMaxNum) - { - unsigned int shiftedNum = blockNum - oldBlockMaxNum; - if (!BlockSetOps::IsMember(comp, newBlocksInLoop, shiftedNum)) - { - BlockSetOps::AddElemD(comp, newBlocksInLoop, shiftedNum); - return false; - } - } - else - { - if (!BlockSetOps::IsMember(comp, oldBlocksInLoop, blockNum)) - { - BlockSetOps::AddElemD(comp, oldBlocksInLoop, blockNum); - return false; - } - } - return true; - } - }; - - LoopBlockSet loopBlocks; // Set of blocks identified as part of the loop - Compiler* comp; - - // See LoopSearch class comment header for a diagram relating these fields: - BasicBlock* head; // Predecessor of unique entry edge - BasicBlock* top; // Successor of back-edge from BOTTOM - BasicBlock* bottom; // Predecessor of back-edge to TOP, also lexically last in-loop block - BasicBlock* entry; // Successor of unique entry edge - - BasicBlock* lastExit; // Most recently discovered exit block - unsigned char exitCount; // Number of discovered exit edges - unsigned int oldBlockMaxNum; // Used to identify new blocks created during compaction - BlockSet bottomBlocks; // BOTTOM blocks of already-recorded loops -#ifdef DEBUG - bool forgotExit = false; // Flags a rare case where lastExit gets nulled out, for assertions -#endif - bool changedFlowGraph = false; // Signals that loop compaction has modified the flow graph - -public: - LoopSearch(Compiler* comp) - : loopBlocks(comp), comp(comp), oldBlockMaxNum(comp->fgBBNumMax), bottomBlocks(BlockSetOps::MakeEmpty(comp)) - { - // Make sure we've renumbered such that the bitsets can hold all the bits - assert(comp->fgBBNumMax <= comp->fgCurBBEpochSize); - } - - //------------------------------------------------------------------------ - // RecordLoop: Notify the Compiler that a loop has been found. - // - // Return Value: - // true - Loop successfully recorded. - // false - Compiler has run out of loop descriptors; loop not recorded. - // - bool RecordLoop() - { - // At this point we have a compact loop - record it in the loop table. - // If we found only one exit, record it in the table too - // (otherwise an exit = nullptr in the loop table means multiple exits). - - BasicBlock* onlyExit = (exitCount == 1 ? lastExit : nullptr); - if (comp->optRecordLoop(head, top, entry, bottom, onlyExit, exitCount)) - { - // Record the BOTTOM block for future reference before returning. - assert(bottom->bbNum <= oldBlockMaxNum); - BlockSetOps::AddElemD(comp, bottomBlocks, bottom->bbNum); - return true; - } - - // Unable to record this loop because the loop descriptor table overflowed. - return false; - } - - //------------------------------------------------------------------------ - // ChangedFlowGraph: Determine whether loop compaction has modified the flow graph. - // - // Return Value: - // true - The flow graph has been modified; fgUpdateChangedFlowGraph should - // be called (which is the caller's responsibility). - // false - The flow graph has not been modified by this LoopSearch. - // - bool ChangedFlowGraph() - { - return changedFlowGraph; - } - - //------------------------------------------------------------------------ - // FindLoop: Search for a loop with the given HEAD block and back-edge. - // - // Arguments: - // head - Block to be the HEAD of any loop identified - // top - Block to be the TOP of any loop identified - // bottom - Block to be the BOTTOM of any loop identified - // - // Return Value: - // true - Found a valid loop. - // false - Did not find a valid loop. - // - // Notes: - // May modify flow graph to make loop compact before returning. - // Will set instance fields to track loop's extent and exits if a valid - // loop is found, and potentially trash them otherwise. - // - bool FindLoop(BasicBlock* head, BasicBlock* top, BasicBlock* bottom) - { - // Is this a loop candidate? - We look for "back edges", i.e. an edge from BOTTOM - // to TOP (note that this is an abuse of notation since this is not necessarily a back edge - // as the definition says, but merely an indication that we have a loop there). - // Thus, we have to be very careful and after entry discovery check that it is indeed - // the only place we enter the loop (especially for non-reducible flow graphs). - - JITDUMP("FindLoop: checking head:" FMT_BB " top:" FMT_BB " bottom:" FMT_BB "\n", head->bbNum, top->bbNum, - bottom->bbNum); - - if (top->bbNum > bottom->bbNum) // is this a backward edge? (from BOTTOM to TOP) - { - // Edge from BOTTOM to TOP is not a backward edge - JITDUMP(" " FMT_BB "->" FMT_BB " is not a backedge\n", bottom->bbNum, top->bbNum); - return false; - } - - if (bottom->bbNum > oldBlockMaxNum) - { - // Not a true back-edge; bottom is a block added to reconnect fall-through during - // loop processing, so its block number does not reflect its position. - JITDUMP(" " FMT_BB "->" FMT_BB " is not a true backedge\n", bottom->bbNum, top->bbNum); - return false; - } - - if (bottom->KindIs(BBJ_EHFINALLYRET, BBJ_EHFAULTRET, BBJ_EHFILTERRET, BBJ_EHCATCHRET, BBJ_CALLFINALLY, - BBJ_SWITCH)) - { - JITDUMP(" bottom odd jump kind\n"); - // BBJ_EHFINALLYRET, BBJ_EHFAULTRET, BBJ_EHFILTERRET, BBJ_EHCATCHRET, and BBJ_CALLFINALLY can never form a - // loop. - // BBJ_SWITCH that has a backward jump appears only for labeled break. - return false; - } - - // The presence of a "back edge" is an indication that a loop might be present here. - // - // Definition: A loop is: - // 1. A collection of STRONGLY CONNECTED nodes i.e. there is a path from any - // node in the loop to any other node in the loop (wholly within the loop) - // 2. The loop has a unique ENTRY, i.e. there is only one way to reach a node - // in the loop from outside the loop, and that is through the ENTRY - - // Let's find the loop ENTRY - BasicBlock* entry = FindEntry(head, top, bottom); - - if (entry == nullptr) - { - // For now, we only recognize loops where HEAD has some successor ENTRY in the loop. - JITDUMP(" can't find entry\n"); - return false; - } - - // Passed the basic checks; initialize instance state for this back-edge. - this->head = head; - this->top = top; - this->entry = entry; - this->bottom = bottom; - this->lastExit = nullptr; - this->exitCount = 0; - - if (!HasSingleEntryCycle()) - { - // There isn't actually a loop between TOP and BOTTOM - JITDUMP(" not single entry cycle\n"); - return false; - } - - if (!loopBlocks.IsMember(top->bbNum)) - { - // The "back-edge" we identified isn't actually part of the flow cycle containing ENTRY - JITDUMP(" top not in loop\n"); - return false; - } - - // Disqualify loops where the first block of the loop is less nested in EH than - // the bottom block. That is, we don't want to handle loops where the back edge - // goes from within an EH region to a first block that is outside that same EH - // region. Note that we *do* handle loops where the first block is the *first* - // block of a more nested EH region (since it is legal to branch to the first - // block of an immediately more nested EH region). So, for example, disqualify - // this: - // - // BB02 - // ... - // try { - // ... - // BB10 BBJ_COND => BB02 - // ... - // } - // - // Here, BB10 is more nested than BB02. - - if (bottom->hasTryIndex() && !comp->bbInTryRegions(bottom->getTryIndex(), top)) - { - JITDUMP("Loop 'top' " FMT_BB " is in an outer EH region compared to loop 'bottom' " FMT_BB ". Rejecting " - "loop.\n", - top->bbNum, bottom->bbNum); - return false; - } - - // We have a valid loop. - return true; - } - - //------------------------------------------------------------------------ - // GetExitCount: Return the exit count computed for the loop - // - unsigned char GetExitCount() const - { - return exitCount; - } - -private: - //------------------------------------------------------------------------ - // FindEntry: See if given HEAD flows to valid ENTRY between given TOP and BOTTOM - // - // Arguments: - // head - Block to be the HEAD of any loop identified - // top - Block to be the TOP of any loop identified - // bottom - Block to be the BOTTOM of any loop identified - // - // Return Value: - // Block to be the ENTRY of any loop identified, or nullptr if no - // such entry meeting our criteria can be found. - // - // Notes: - // Returns main entry if one is found, does not check for side-entries. - // - BasicBlock* FindEntry(BasicBlock* head, BasicBlock* top, BasicBlock* bottom) - { - if (head->KindIs(BBJ_ALWAYS)) - { - if (head->GetTarget()->bbNum <= bottom->bbNum && head->GetTarget()->bbNum >= top->bbNum) - { - // OK - we enter somewhere within the loop. - return head->GetTarget(); - } - else - { - // special case - don't consider now - // assert (!"Loop entered in weird way!"); - return nullptr; - } - } - // Can we fall through into the loop? - else if (head->KindIs(BBJ_COND)) - { - // The ENTRY is at the TOP (a do-while loop) - return top; - } - else - { - return nullptr; // HEAD does not flow into the loop; bail for now - } - } - - //------------------------------------------------------------------------ - // HasSingleEntryCycle: Perform a reverse flow walk from ENTRY, visiting - // only blocks between TOP and BOTTOM, to determine if such a cycle - // exists and if it has a single entry. - // - // Return Value: - // true - Found a single-entry cycle. - // false - Did not find a single-entry cycle. - // - // Notes: - // Will mark (in `loopBlocks`) all blocks found to participate in the cycle. - // - bool HasSingleEntryCycle() - { - // Now do a backwards flow walk from entry to see if we have a single-entry loop - bool foundCycle = false; - - // Seed the loop block set and worklist with the entry block. - loopBlocks.Reset(entry->bbNum); - jitstd::list worklist(comp->getAllocator(CMK_LoopOpt)); - worklist.push_back(entry); - - while (!worklist.empty()) - { - BasicBlock* block = worklist.back(); - worklist.pop_back(); - - // Make sure ENTRY dominates all blocks in the loop. - if (block->bbNum > oldBlockMaxNum) - { - // This is a new block we added to connect fall-through, so the - // recorded dominator information doesn't cover it. Just continue, - // and when we process its unique predecessor we'll abort if ENTRY - // doesn't dominate that. - } - else if (!comp->m_domTree->Dominates(entry, block)) - { - JITDUMP(" (find cycle) entry:" FMT_BB " does not dominate " FMT_BB "\n", entry->bbNum, block->bbNum); - return false; - } - - // Add preds to the worklist, checking for side-entries. - for (BasicBlock* const predBlock : block->PredBlocks()) - { - unsigned int testNum = PositionNum(predBlock); - - if ((testNum < top->bbNum) || (testNum > bottom->bbNum)) - { - // Pred is out of loop range - if (block == entry) - { - if (predBlock == head) - { - // This is the single entry we expect. - continue; - } - // ENTRY has some pred other than head outside the loop. If ENTRY does not - // dominate this pred, we'll consider this a side-entry and skip this loop; - // otherwise the loop is still valid and this may be a (flow-wise) back-edge - // of an outer loop. For the dominance test, if `predBlock` is a new block, use - // its unique predecessor since the dominator tree has info for that. - BasicBlock* effectivePred = (predBlock->bbNum > oldBlockMaxNum ? predBlock->Prev() : predBlock); - if (comp->m_domTree->Dominates(entry, effectivePred)) - { - // Outer loop back-edge - continue; - } - } - - // There are multiple entries to this loop, don't consider it. - - JITDUMP(" (find cycle) multiple entry:" FMT_BB "\n", block->bbNum); - return false; - } - - bool isFirstVisit; - if (predBlock == entry) - { - // We have indeed found a cycle in the flow graph. - JITDUMP(" (find cycle) found cycle\n"); - isFirstVisit = !foundCycle; - foundCycle = true; - assert(loopBlocks.IsMember(predBlock->bbNum)); - } - else if (loopBlocks.TestAndInsert(predBlock->bbNum)) - { - // Already visited this pred - isFirstVisit = false; - } - else - { - // Add this predBlock to the worklist - worklist.push_back(predBlock); - isFirstVisit = true; - } - - if (isFirstVisit && !predBlock->IsLast() && (PositionNum(predBlock->Next()) == predBlock->bbNum)) - { - // We've created a new block immediately after `predBlock` to - // reconnect what was fall-through. Mark it as in-loop also; - // it needs to stay with `prev` and if it exits the loop we'd - // just need to re-create it if we tried to move it out. - loopBlocks.Insert(predBlock->Next()->bbNum); - } - } - } - - return foundCycle; - } - - //------------------------------------------------------------------------ - // PositionNum: Get the number identifying a block's position per the - // lexical ordering that existed before searching for (and compacting) - // loops. - // - // Arguments: - // block - Block whose position is desired. - // - // Return Value: - // A number indicating that block's position relative to others. - // - // Notes: - // When the given block is a new one created during loop compaction, - // the number of its unique predecessor is returned. - // - unsigned int PositionNum(BasicBlock* block) - { - if (block->bbNum > oldBlockMaxNum) - { - // This must be a block we inserted to connect fall-through after moving blocks. - // To determine if it's in the loop or not, use the number of its unique predecessor - // block. - assert(block->PrevIs(block->bbPreds->getSourceBlock())); - assert(block->bbPreds->getNextPredEdge() == nullptr); - return block->Prev()->bbNum; - } - return block->bbNum; - } -}; -} // end (anonymous) namespace - -//------------------------------------------------------------------------ -// optFindNaturalLoops: Find the natural loops, using dominators. Note that the test for -// a loop is slightly different from the standard one, because we have not done a depth -// first reordering of the basic blocks. -// -// See LoopSearch class comment header for a description of the loops found. -// -// We will find and record a maximum of BasicBlock::MAX_LOOP_NUM loops (currently 64). -// -void Compiler::optFindNaturalLoops() -{ -#ifdef DEBUG - if (verbose) - { - printf("*************** In optFindNaturalLoops()\n"); - } -#endif // DEBUG - - assert(fgHasLoops); - -#if COUNT_LOOPS - hasMethodLoops = false; - loopsThisMethod = 0; - loopOverflowThisMethod = false; -#endif - - LoopSearch search(this); - - // TODO-Quirk: Remove - BasicBlock* first = fgCanonicalizedFirstBB ? fgFirstBB->Next() : fgFirstBB; - - for (BasicBlock* head = first; !head->IsLast(); head = head->Next()) - { - BasicBlock* top = head->Next(); - - // Blocks that are rarely run have a zero bbWeight and should never be optimized here. - if (top->bbWeight == BB_ZERO_WEIGHT) - { - continue; - } - - for (BasicBlock* const predBlock : top->PredBlocks()) - { - if (search.FindLoop(head, top, predBlock)) - { - // Found a loop; record it and see if we've hit the limit. - bool recordedLoop = search.RecordLoop(); - - (void)recordedLoop; // avoid unusued variable warnings in COUNT_LOOPS and !DEBUG - -#if COUNT_LOOPS - if (!hasMethodLoops) - { - // Mark the method as containing natural loops - totalLoopMethods++; - hasMethodLoops = true; - } - - // Increment total number of loops found - totalLoopCount++; - loopsThisMethod++; - - // Keep track of the number of exits - loopExitCountTable.record(static_cast(search.GetExitCount())); - - // Note that we continue to look for loops even if - // (optLoopCount == BasicBlock::MAX_LOOP_NUM), in contrast to the !COUNT_LOOPS code below. - // This gives us a better count and stats. Hopefully it doesn't affect actual codegen. - CLANG_FORMAT_COMMENT_ANCHOR; - -#else // COUNT_LOOPS - assert(recordedLoop); - if (optLoopCount == BasicBlock::MAX_LOOP_NUM) - { - // We won't be able to record any more loops, so stop looking. - goto NO_MORE_LOOPS; - } -#endif // COUNT_LOOPS - - // Continue searching preds of `top` to see if any other are - // back-edges (this can happen for nested loops). The iteration - // is safe because the compaction we do only modifies predecessor - // lists of blocks that gain or lose fall-through from their - // `bbPrev`, but since the motion is from within the loop to below - // it, we know we're not altering the relationship between `top` - // and its `bbPrev`. - } - } - } - -#if !COUNT_LOOPS -NO_MORE_LOOPS: -#endif // !COUNT_LOOPS - -#if COUNT_LOOPS - loopCountTable.record(loopsThisMethod); - if (maxLoopsPerMethod < loopsThisMethod) - { - maxLoopsPerMethod = loopsThisMethod; - } - if (loopOverflowThisMethod) - { - totalLoopOverflows++; - } -#endif // COUNT_LOOPS - - bool mod = search.ChangedFlowGraph(); - - if (mod) - { - fgInvalidateDfsTree(); - fgRenumberBlocks(); - m_dfsTree = fgComputeDfs(); - m_domTree = FlowGraphDominatorTree::Build(m_dfsTree); - } - - // Now the loop indices are stable. We can figure out parent/child relationships - // (using table indices to name loops), and label blocks. - for (unsigned char loopInd = 1; loopInd < optLoopCount; loopInd++) - { - for (unsigned char possibleParent = loopInd; possibleParent > 0;) - { - possibleParent--; - if (optLoopTable[possibleParent].lpContains(optLoopTable[loopInd])) - { - optLoopTable[loopInd].lpParent = possibleParent; - optLoopTable[loopInd].lpSibling = optLoopTable[possibleParent].lpChild; - optLoopTable[possibleParent].lpChild = loopInd; - break; - } - } - } - - // Now label the blocks with the innermost loop to which they belong. Since parents - // precede children in the table, doing the labeling for each loop in order will achieve - // this -- the innermost loop labeling will be done last. (Inner loop blocks will be - // labeled multiple times before being correct at the end.) - for (unsigned char loopInd = 0; loopInd < optLoopCount; loopInd++) - { - for (BasicBlock* const blk : optLoopTable[loopInd].LoopBlocks()) - { - blk->bbNatLoopNum = loopInd; - } - } - - for (BasicBlock* block : Blocks()) - { - block->RemoveFlags(BBF_OLD_LOOP_HEADER_QUIRK); - } - - for (unsigned loopInd = 0; loopInd < optLoopCount; loopInd++) - { - optLoopTable[loopInd].lpEntry->SetFlags(BBF_OLD_LOOP_HEADER_QUIRK); - } - -#ifdef DEBUG - if (verbose && (optLoopCount > 0)) - { - optPrintLoopTable(); - } -#endif // DEBUG } +#endif // DEBUG + //------------------------------------------------------------------------ // optRedirectBlock: Replace the branch successors of a block based on a block map. // @@ -2255,97 +707,6 @@ void Compiler::optRedirectBlock(BasicBlock* blk, BlockToBlockMap* redirectMap, R } } -// Returns true if 'block' is an entry block for any loop in 'optLoopTable' -bool Compiler::optIsLoopEntry(BasicBlock* block) const -{ - for (unsigned char loopInd = 0; loopInd < optLoopCount; loopInd++) - { - if (optLoopTable[loopInd].lpIsRemoved()) - { - continue; - } - - if (optLoopTable[loopInd].lpEntry == block) - { - return true; - } - } - return false; -} - -//----------------------------------------------------------------------------- -// optLoopContains: Check if one loop contains another -// -// Arguments: -// l1 -- loop num of containing loop (must be valid loop num) -// l2 -- loop num of contained loop (valid loop num, or NOT_IN_LOOP) -// -// Returns: -// True if loop described by l2 is contained within l1. -// -// Notes: -// A loop contains itself. -// -bool Compiler::optLoopContains(unsigned l1, unsigned l2) const -{ - assert(l1 < optLoopCount); - assert((l2 < optLoopCount) || (l2 == BasicBlock::NOT_IN_LOOP)); - - if (l1 == l2) - { - return true; - } - else if (l2 == BasicBlock::NOT_IN_LOOP) - { - return false; - } - else - { - return optLoopContains(l1, optLoopTable[l2].lpParent); - } -} - -//----------------------------------------------------------------------------- -// optLoopEntry: For a given preheader of a loop, returns the lpEntry. -// -// Arguments: -// preHeader -- preheader of a loop -// -// Returns: -// Corresponding loop entry block. -// -BasicBlock* Compiler::optLoopEntry(BasicBlock* preHeader) -{ - assert(preHeader->HasFlag(BBF_LOOP_PREHEADER)); - assert(preHeader->KindIs(BBJ_ALWAYS)); - return preHeader->GetTarget(); -} - -//----------------------------------------------------------------------------- -// optUpdateLoopHead: Replace the `head` block of a loop in the loop table. -// Considers all child loops that might share the same head (recursively). -// -// Arguments: -// loopInd -- loop num of loop -// from -- current loop head block -// to -- replacement loop head block -// -void Compiler::optUpdateLoopHead(unsigned loopInd, BasicBlock* from, BasicBlock* to) -{ - assert(optLoopTable[loopInd].lpHead == from); - JITDUMP("Replace " FMT_LP " head " FMT_BB " with " FMT_BB "\n", loopInd, from->bbNum, to->bbNum); - optLoopTable[loopInd].lpHead = to; - for (unsigned char childLoop = optLoopTable[loopInd].lpChild; // - childLoop != BasicBlock::NOT_IN_LOOP; // - childLoop = optLoopTable[childLoop].lpSibling) - { - if (optLoopTable[childLoop].lpHead == from) - { - optUpdateLoopHead(childLoop, from, to); - } - } -} - //----------------------------------------------------------------------------- // optIterSmallOverflow: Helper for loop unrolling. Determine if "i += const" will // cause an overflow exception for the small types. @@ -4143,8 +2504,8 @@ void Compiler::optMarkLoopHeads() } //----------------------------------------------------------------------------- -// optResetLoopInfo: reset all loop info in preparation for rebuilding the loop table, or preventing -// future phases from accessing loop-related data. +// optResetLoopInfo: reset all loop info in preparation for refinding the loops +// and scaling blocks based on it. // void Compiler::optResetLoopInfo() { @@ -4155,16 +2516,6 @@ void Compiler::optResetLoopInfo() } #endif - optLoopCount = 0; // This will force the table to be rebuilt - - // This will cause users to crash if they use the table when it is considered empty. - // TODO: the loop table is always allocated as the same (maximum) size, so this is wasteful. - // We could zero it out (possibly only in DEBUG) to be paranoid, but there's no reason to - // force it to be re-allocated. - optLoopTable = nullptr; - optLoopTableValid = false; - optLoopsRequirePreHeaders = false; - for (BasicBlock* const block : Blocks()) { // If the block weight didn't come from profile data, reset it so it can be calculated again. @@ -4175,7 +2526,6 @@ void Compiler::optResetLoopInfo() } block->RemoveFlags(BBF_LOOP_FLAGS); - block->bbNatLoopNum = BasicBlock::NOT_IN_LOOP; } } @@ -4283,7 +2633,7 @@ void Compiler::optFindAndScaleGeneralLoopBlocks() // optFindLoops: find loops in the function. // // The JIT recognizes two types of loops in a function: natural loops and "general" (or "unnatural") loops. -// Natural loops are those which get added to the loop table. Most downstream optimizations require +// Natural loops are those which get added to Compiler::m_loops. Most downstream optimizations require // using natural loops. See `FlowGraphNaturalLoop` for a definition of the criteria satisfied by a natural loop. // A general loop is defined as a lexical (program order) range of blocks where a later block branches to an // earlier block (that is, there is a back edge in the flow graph), and the later block is reachable from the earlier @@ -4292,56 +2642,34 @@ void Compiler::optFindAndScaleGeneralLoopBlocks() // Notes: // Also (re)sets all non-IBC block weights. // -void Compiler::optFindLoops() +PhaseStatus Compiler::optFindLoopsPhase() { #ifdef DEBUG if (verbose) { - printf("*************** In optFindLoops()\n"); + printf("*************** In optFindLoopsPhase()\n"); } #endif - noway_assert(opts.OptimizationEnabled()); - optMarkLoopHeads(); - // Were there any potential loops in the flow graph? - - if (fgHasLoops) - { - optFindNaturalLoops(); - } - - optLoopTableValid = true; -} - -//----------------------------------------------------------------------------- -// optFindLoopsPhase: The wrapper function for the "find loops" phase. -// -PhaseStatus Compiler::optFindLoopsPhase() -{ + assert(m_dfsTree != nullptr); optFindLoops(); - m_dfsTree = fgComputeDfs(); - optFindNewLoops(); - if (fgHasLoops) { optFindAndScaleGeneralLoopBlocks(); } - // The old loop table is no longer valid. - optLoopTableValid = false; - optLoopTable = nullptr; - optLoopCount = 0; + optNumNaturalLoopsFound = (unsigned)m_loops->NumLoops(); return PhaseStatus::MODIFIED_EVERYTHING; } //----------------------------------------------------------------------------- -// optFindNewLoops: Compute new loops and cross validate with old loop table. +// optFindLoops: Find, compact and canonicalize natural loops. // -void Compiler::optFindNewLoops() +void Compiler::optFindLoops() { m_loops = FlowGraphNaturalLoops::Find(m_dfsTree); @@ -4898,54 +3226,6 @@ void Compiler::optSetPreheaderWeight(FlowGraphNaturalLoop* loop, BasicBlock* pre } } -/***************************************************************************** - * - * Determine the kind of interference for the call. - */ - -/* static */ inline Compiler::callInterf Compiler::optCallInterf(GenTreeCall* call) -{ - // if not a helper, kills everything - if (call->gtCallType != CT_HELPER) - { - return CALLINT_ALL; - } - - // setfield and array address store kill all indirections - switch (eeGetHelperNum(call->gtCallMethHnd)) - { - case CORINFO_HELP_ASSIGN_REF: // Not strictly needed as we don't make a GT_CALL with this - case CORINFO_HELP_CHECKED_ASSIGN_REF: // Not strictly needed as we don't make a GT_CALL with this - case CORINFO_HELP_ASSIGN_BYREF: // Not strictly needed as we don't make a GT_CALL with this - case CORINFO_HELP_SETFIELDOBJ: - case CORINFO_HELP_ARRADDR_ST: - - return CALLINT_REF_INDIRS; - - case CORINFO_HELP_SETFIELDFLOAT: - case CORINFO_HELP_SETFIELDDOUBLE: - case CORINFO_HELP_SETFIELD8: - case CORINFO_HELP_SETFIELD16: - case CORINFO_HELP_SETFIELD32: - case CORINFO_HELP_SETFIELD64: - - return CALLINT_SCL_INDIRS; - - case CORINFO_HELP_ASSIGN_STRUCT: // Not strictly needed as we don't use this - case CORINFO_HELP_MEMSET: // Not strictly needed as we don't make a GT_CALL with this - case CORINFO_HELP_MEMCPY: // Not strictly needed as we don't make a GT_CALL with this - case CORINFO_HELP_SETFIELDSTRUCT: - - return CALLINT_ALL_INDIRS; - - default: - break; - } - - // other helpers kill nothing - return CALLINT_NONE; -} - /***************************************************************************** * * See if the given tree can be computed in the given precision (which must @@ -5219,390 +3499,96 @@ bool Compiler::optNarrowTree(GenTree* tree, var_types srct, var_types dstt, Valu { if (tree->gtOper == GT_MUL && (tree->gtFlags & GTF_MUL_64RSLT)) { - tree->gtFlags &= ~GTF_MUL_64RSLT; - } - - tree->gtType = genActualType(dstt); - tree->SetVNs(vnpNarrow); - } - - return true; - - case GT_IND: - - NARROW_IND: - - if ((dstSize > genTypeSize(tree->gtType)) && - (varTypeIsUnsigned(dstt) && !varTypeIsUnsigned(tree->gtType))) - { - return false; - } - - /* Simply change the type of the tree */ - - if (doit && (dstSize <= genTypeSize(tree->gtType))) - { - if (!varTypeIsSmall(dstt)) - { - dstt = varTypeToSigned(dstt); - } - - tree->gtType = dstt; - tree->SetVNs(vnpNarrow); - } - - return true; - - case GT_EQ: - case GT_NE: - case GT_LT: - case GT_LE: - case GT_GT: - case GT_GE: - - /* These can always be narrowed since they only represent 0 or 1 */ - return true; - - case GT_CAST: - { -#ifdef DEBUG - if ((tree->gtDebugFlags & GTF_DEBUG_CAST_DONT_FOLD) != 0) - { - return false; - } -#endif - - if ((tree->CastToType() != srct) || tree->gtOverflow()) - { - return false; - } - - if (varTypeIsInt(op1) && varTypeIsInt(dstt) && tree->TypeIs(TYP_LONG)) - { - // We have a CAST that converts into to long while dstt is int. - // so we can just convert the cast to int -> int and someone will clean it up. - if (doit) - { - tree->CastToType() = TYP_INT; - tree->ChangeType(TYP_INT); - tree->ClearUnsigned(); - } - return true; - } - } - return false; - - case GT_COMMA: - if (!gtIsActiveCSE_Candidate(op2) && optNarrowTree(op2, srct, dstt, vnpNarrow, doit)) - { - /* Simply change the type of the tree */ - - if (doit) - { - tree->gtType = genActualType(dstt); - tree->SetVNs(vnpNarrow); - } - return true; - } - return false; - - default: - noway_assert(doit == false); - return false; - } - } - - return false; -} - -//------------------------------------------------------------------------ -// optIsVarAssignedWithDesc: do a walk to record local modification data for a statement -// -// Arguments: -// stmt - the statement to walk -// dsc - [in, out] data for the walk -// -bool Compiler::optIsVarAssignedWithDesc(Statement* stmt, isVarAssgDsc* dsc) -{ - class IsVarAssignedVisitor : public GenTreeVisitor - { - isVarAssgDsc* m_dsc; - - public: - enum - { - DoPreOrder = true, - UseExecutionOrder = true, - }; - - IsVarAssignedVisitor(Compiler* comp, isVarAssgDsc* dsc) : GenTreeVisitor(comp), m_dsc(dsc) - { - } - - fgWalkResult PreOrderVisit(GenTree** use, GenTree* user) - { - GenTree* const tree = *use; - - if (tree->OperIs(GT_STOREIND)) - { - // Set the proper indirection bits. - varRefKinds refs = varTypeIsGC(tree) ? VR_IND_REF : VR_IND_SCL; - m_dsc->ivaMaskInd = varRefKinds(m_dsc->ivaMaskInd | refs); - } - - // Can this tree define a local? - // - if (!tree->OperIsSsaDef()) - { - return WALK_CONTINUE; - } - - // Determine what's written and check for calls. - // - if (tree->OperIs(GT_CALL)) - { - m_dsc->ivaMaskCall = optCallInterf(tree->AsCall()); - } - else if (tree->OperIs(GT_STORE_LCL_FLD)) - { - // We can't track every field of every var. Moreover, indirections - // may access different parts of the var as different (but overlapping) - // fields. So just treat them as indirect accesses. - varRefKinds refs = varTypeIsGC(tree->TypeGet()) ? VR_IND_REF : VR_IND_SCL; - m_dsc->ivaMaskInd = varRefKinds(m_dsc->ivaMaskInd | refs); - } - - // Determine if the tree modifies a particular local - // - GenTreeLclVarCommon* lcl = nullptr; - if (tree->DefinesLocal(m_compiler, &lcl)) - { - const unsigned lclNum = lcl->GetLclNum(); - - if (lclNum < lclMAX_ALLSET_TRACKED) - { - AllVarSetOps::AddElemD(m_compiler, m_dsc->ivaMaskVal, lclNum); - } - else - { - m_dsc->ivaMaskIncomplete = true; - } - - // Bail out if we were checking for one particular local - // and we now see it's modified (ignoring perhaps - // the one tree where we expect modifications). - // - if ((lclNum == m_dsc->ivaVar) && (tree != m_dsc->ivaSkip)) - { - return WALK_ABORT; - } - } - - return WALK_CONTINUE; - } - }; - - IsVarAssignedVisitor walker(this, dsc); - return walker.WalkTree(stmt->GetRootNodePointer(), nullptr) != WALK_CONTINUE; -} - -//------------------------------------------------------------------------ -// optIsVarAssigned: see if a local is assigned in a range of blocks -// -// Arguments: -// beg - first block in range -// end - last block in range -// skip - tree to ignore (nullptr if none) -// var - local to check -// -// Returns: -// true if local is directly modified -// -// Notes: -// Does a full walk of all blocks/statements/trees, so potentially expensive. -// -// Does not do proper checks for struct fields or exposed locals. -// -bool Compiler::optIsVarAssigned(BasicBlock* beg, BasicBlock* end, GenTree* skip, unsigned var) -{ - isVarAssgDsc desc; - - desc.ivaSkip = skip; - desc.ivaVar = var; - desc.ivaMaskCall = CALLINT_NONE; - AllVarSetOps::AssignNoCopy(this, desc.ivaMaskVal, AllVarSetOps::MakeEmpty(this)); + tree->gtFlags &= ~GTF_MUL_64RSLT; + } - for (;;) - { - noway_assert(beg != nullptr); + tree->gtType = genActualType(dstt); + tree->SetVNs(vnpNarrow); + } - for (Statement* const stmt : beg->Statements()) - { - if (optIsVarAssignedWithDesc(stmt, &desc)) - { return true; - } - } - - if (beg == end) - { - break; - } - beg = beg->Next(); - } + case GT_IND: - return false; -} + NARROW_IND: -//------------------------------------------------------------------------ -// optIsVarAssgLoop: see if a local is assigned in a loop -// -// Arguments: -// lnum - loop number -// var - var to check -// -// Returns: -// true if var can possibly be modified in the loop specified by lnum -// false if var is a loop invariant -// -bool Compiler::optIsVarAssgLoop(unsigned lnum, unsigned var) -{ - assert(lnum < optLoopCount); + if ((dstSize > genTypeSize(tree->gtType)) && + (varTypeIsUnsigned(dstt) && !varTypeIsUnsigned(tree->gtType))) + { + return false; + } - LclVarDsc* const varDsc = lvaGetDesc(var); - if (varDsc->IsAddressExposed()) - { - // Assume the worst (that var is possibly modified in the loop) - // - return true; - } + /* Simply change the type of the tree */ - if (var < lclMAX_ALLSET_TRACKED) - { - ALLVARSET_TP vs(AllVarSetOps::MakeSingleton(this, var)); + if (doit && (dstSize <= genTypeSize(tree->gtType))) + { + if (!varTypeIsSmall(dstt)) + { + dstt = varTypeToSigned(dstt); + } - // If local is a promoted field, also check for modifications to parent. - // - if (varDsc->lvIsStructField) - { - unsigned const parentVar = varDsc->lvParentLcl; - assert(!lvaGetDesc(parentVar)->IsAddressExposed()); - assert(lvaGetDesc(parentVar)->lvPromoted); + tree->gtType = dstt; + tree->SetVNs(vnpNarrow); + } - if (parentVar < lclMAX_ALLSET_TRACKED) - { - JITDUMP("optIsVarAssgLoop: V%02u promoted, also checking V%02u\n", var, parentVar); - AllVarSetOps::AddElemD(this, vs, parentVar); - } - else - { - // Parent var index is too large, assume the worst. - // return true; - } - } - - return optIsSetAssgLoop(lnum, vs) != 0; - } - else - { - if (varDsc->lvIsStructField) - { - return true; - } - - return optIsVarAssigned(optLoopTable[lnum].lpHead->Next(), optLoopTable[lnum].lpBottom, nullptr, var); - } -} -//------------------------------------------------------------------------ -// optIsSetAssgLoop: see if a set of locals is assigned in a loop -// -// Arguments: -// lnum - loop number -// vars - var set to check -// inds - also consider impact of indirect stores and calls -// -// Returns: -// true if any of vars are possibly modified in any of the blocks of the -// loop specified by lnum, or if the loop contains any of the specified -// aliasing operations. -// -// Notes: -// Uses a cache to avoid repeatedly scanning the loop blocks. However this -// cache never invalidates and so this method must be used with care. -// -bool Compiler::optIsSetAssgLoop(unsigned lnum, ALLVARSET_VALARG_TP vars, varRefKinds inds) -{ - noway_assert(lnum < optLoopCount); - LoopDsc* loop = &optLoopTable[lnum]; + case GT_EQ: + case GT_NE: + case GT_LT: + case GT_LE: + case GT_GT: + case GT_GE: - // Do we already know what variables are assigned within this loop? - // - if (!(loop->lpFlags & LPFLG_ASGVARS_YES)) - { - isVarAssgDsc desc; + /* These can always be narrowed since they only represent 0 or 1 */ + return true; - // Prepare the descriptor used by the tree walker call-back - // - desc.ivaVar = (unsigned)-1; - desc.ivaSkip = nullptr; - AllVarSetOps::AssignNoCopy(this, desc.ivaMaskVal, AllVarSetOps::MakeEmpty(this)); - desc.ivaMaskInd = VR_NONE; - desc.ivaMaskCall = CALLINT_NONE; - desc.ivaMaskIncomplete = false; - - // Now walk all the statements of the loop - // - for (BasicBlock* const block : loop->LoopBlocks()) - { - for (Statement* const stmt : block->NonPhiStatements()) + case GT_CAST: { - optIsVarAssignedWithDesc(stmt, &desc); - - if (desc.ivaMaskIncomplete) +#ifdef DEBUG + if ((tree->gtDebugFlags & GTF_DEBUG_CAST_DONT_FOLD) != 0) { - loop->lpFlags |= LPFLG_ASGVARS_INC; + return false; } - } - } +#endif - AllVarSetOps::Assign(this, loop->lpAsgVars, desc.ivaMaskVal); - loop->lpAsgInds = desc.ivaMaskInd; - loop->lpAsgCall = desc.ivaMaskCall; + if ((tree->CastToType() != srct) || tree->gtOverflow()) + { + return false; + } - // Now we know what variables are assigned in the loop - // - loop->lpFlags |= LPFLG_ASGVARS_YES; - } + if (varTypeIsInt(op1) && varTypeIsInt(dstt) && tree->TypeIs(TYP_LONG)) + { + // We have a CAST that converts into to long while dstt is int. + // so we can just convert the cast to int -> int and someone will clean it up. + if (doit) + { + tree->CastToType() = TYP_INT; + tree->ChangeType(TYP_INT); + tree->ClearUnsigned(); + } + return true; + } + } + return false; - // Now we can finally test the caller's mask against the loop's - // - if (!AllVarSetOps::IsEmptyIntersection(this, loop->lpAsgVars, vars) || (loop->lpAsgInds & inds)) - { - return true; - } + case GT_COMMA: + if (!gtIsActiveCSE_Candidate(op2) && optNarrowTree(op2, srct, dstt, vnpNarrow, doit)) + { + /* Simply change the type of the tree */ - // If caller is worried about possible indirect effects, check - // what we know about the calls in the loop. - // - if (inds != 0) - { - switch (loop->lpAsgCall) - { - case CALLINT_ALL: - return true; - case CALLINT_REF_INDIRS: - return (inds & VR_IND_REF) != 0; - case CALLINT_SCL_INDIRS: - return (inds & VR_IND_SCL) != 0; - case CALLINT_ALL_INDIRS: - return (inds & (VR_IND_REF | VR_IND_SCL)) != 0; - case CALLINT_NONE: + if (doit) + { + tree->gtType = genActualType(dstt); + tree->SetVNs(vnpNarrow); + } + return true; + } return false; + default: - noway_assert(!"Unexpected lpAsgCall value"); + noway_assert(doit == false); + return false; } } @@ -5874,7 +3860,6 @@ PhaseStatus Compiler::optHoistLoopCode() { printf("\n*************** In optHoistLoopCode()\n"); fgDispHandlerTab(); - optPrintLoopTable(); } #endif @@ -5946,15 +3931,9 @@ bool Compiler::optHoistThisLoop(FlowGraphNaturalLoop* loop, LoopHoistContext* ho #ifdef DEBUG if (verbose) { - printf("optHoistThisLoop for loop " FMT_LP " (head: " FMT_BB "):\n", loop->GetIndex(), - loop->GetHeader()->bbNum); + printf("optHoistThisLoop processing "); + FlowGraphNaturalLoop::Dump(loop); printf(" Loop body %s a call\n", sideEffs.ContainsCall ? "contains" : "does not contain"); - printf(" Loop has %s\n", (loop->ExitEdges().size() == 1) ? "single exit" : "multiple exits"); - printf(" Blocks:\n"); - loop->VisitLoopBlocksReversePostOrder([](BasicBlock* bb) { - printf(" " FMT_BB "\n", bb->bbNum); - return BasicBlockVisit::Continue; - }); } #endif @@ -8175,245 +6154,3 @@ PhaseStatus Compiler::optVNBasedDeadStoreRemoval() return madeChanges ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING; } - -#ifdef DEBUG - -//------------------------------------------------------------------------ -// optAnyChildNotRemoved: Recursively check the child loops of a loop to see if any of them -// are still live (that is, not marked as LPFLG_REMOVED). This check is done when we are -// removing a parent, just to notify that there is something odd about leaving a live child. -// -// Arguments: -// loopNum - the loop number to check -// -bool Compiler::optAnyChildNotRemoved(unsigned loopNum) -{ - assert(loopNum < optLoopCount); - - // Now recursively mark the children. - for (BasicBlock::loopNumber l = optLoopTable[loopNum].lpChild; // - l != BasicBlock::NOT_IN_LOOP; // - l = optLoopTable[l].lpSibling) - { - if (!optLoopTable[l].lpIsRemoved()) - { - return true; - } - - if (optAnyChildNotRemoved(l)) - { - return true; - } - } - - // All children were removed - return false; -} - -#endif // DEBUG - -//------------------------------------------------------------------------ -// optMarkLoopRemoved: Mark the specified loop as removed (some optimization, such as unrolling, has made the -// loop no longer exist). Note that only the given loop is marked as being removed; if it has any children, -// they are not touched (but a warning message is output to the JitDump). -// This method resets the `bbNatLoopNum` field to point to either parent's loop number or NOT_IN_LOOP. -// For consistency, it also updates the child loop's `lpParent` field to have its parent -// -// Arguments: -// loopNum - the loop number to remove -// -void Compiler::optMarkLoopRemoved(unsigned loopNum) -{ -#ifdef DEBUG - if (verbose) - { - printf("Marking loop " FMT_LP " removed\n", loopNum); - optPrintLoopTable(); - } -#endif - - assert(loopNum < optLoopCount); - LoopDsc& loop = optLoopTable[loopNum]; - assert(!loop.lpIsRemoved()); - - for (BasicBlock* const auxBlock : loop.LoopBlocks()) - { - if (auxBlock->bbNatLoopNum == loopNum) - { - JITDUMP("Resetting loop number for " FMT_BB " from " FMT_LP " to " FMT_LP ".\n", auxBlock->bbNum, - auxBlock->bbNatLoopNum, loop.lpParent); - auxBlock->bbNatLoopNum = loop.lpParent; - } - } - - if (loop.lpParent != BasicBlock::NOT_IN_LOOP) - { - // If there is a parent loop, we need to update two things for removed loop `loopNum`: - // 1. Update its siblings so that they no longer point to the `loopNum`. - // 2. Update the children loops to make them point to the parent loop instead. - // - // When we move all the child loops of current loop `loopNum` to its parent, we insert - // those child loops at the same spot where `loopnum` was present in the child chain of - // its parent loop. This is accomplished by updating the existing siblings of `loopNum` - // to now point to the child loops. - // - // If L02 is removed: - // 1. L01's sibling is updated from L02 to L03. - // 2. L03 and L06's parents is updated from L02 to L00. - // 3. L06's sibling is updated from NOT_IN_LOOP to L07. - // - // L00 L00 - // L01 L01 - // L02 => L03 - // L03 L04 - // L04 L05 - // L05 L06 - // L06 L07 - // L07 - // - LoopDsc& parentLoop = optLoopTable[loop.lpParent]; - BasicBlock::loopNumber firstChildLoop = loop.lpChild; - BasicBlock::loopNumber lastChildLoop = BasicBlock::NOT_IN_LOOP; - BasicBlock::loopNumber prevSibling = BasicBlock::NOT_IN_LOOP; - BasicBlock::loopNumber nextSibling = BasicBlock::NOT_IN_LOOP; - for (BasicBlock::loopNumber l = parentLoop.lpChild; l != BasicBlock::NOT_IN_LOOP; l = optLoopTable[l].lpSibling) - { - // We shouldn't see removed loop in loop table. - assert(!optLoopTable[l].lpIsRemoved()); - - nextSibling = optLoopTable[l].lpSibling; - if (l == loopNum) - { - // This condition is not in for-loop just in case there is bad state of loopTable and we - // end up spining infinitely. - break; - } - prevSibling = l; - } - - if (firstChildLoop == BasicBlock::NOT_IN_LOOP) - { - // There are no child loops in `loop`. - // Just update `loop`'s siblings and parentLoop's lpChild, if applicable. - - if (parentLoop.lpChild == loopNum) - { - // If `loop` was the first child - assert(prevSibling == BasicBlock::NOT_IN_LOOP); - - JITDUMP(FMT_LP " has no child loops but is the first child of its parent loop " FMT_LP - ". Update first child to " FMT_LP ".\n", - loopNum, loop.lpParent, nextSibling); - - parentLoop.lpChild = nextSibling; - } - else - { - // `loop` was non-first child - assert(prevSibling != BasicBlock::NOT_IN_LOOP); - - JITDUMP(FMT_LP " has no child loops. Update sibling link " FMT_LP " -> " FMT_LP ".\n", loopNum, - prevSibling, nextSibling); - - optLoopTable[prevSibling].lpSibling = nextSibling; - } - } - else - { - // There are child loops in `loop` that needs to be moved - // under `loop`'s parents. - - if (parentLoop.lpChild == loopNum) - { - // If `loop` was the first child - assert(prevSibling == BasicBlock::NOT_IN_LOOP); - - JITDUMP(FMT_LP " has child loops and is also the first child of its parent loop " FMT_LP - ". Update parent's first child to " FMT_LP ".\n", - loopNum, loop.lpParent, firstChildLoop); - - parentLoop.lpChild = firstChildLoop; - } - else - { - // `loop` was non-first child - assert(prevSibling != BasicBlock::NOT_IN_LOOP); - - JITDUMP(FMT_LP " has child loops. Update sibling link " FMT_LP " -> " FMT_LP ".\n", loopNum, - prevSibling, firstChildLoop); - - optLoopTable[prevSibling].lpSibling = firstChildLoop; - } - - // Update lpParent of all child loops - for (BasicBlock::loopNumber l = firstChildLoop; l != BasicBlock::NOT_IN_LOOP; l = optLoopTable[l].lpSibling) - { - assert(!optLoopTable[l].lpIsRemoved()); - - if (optLoopTable[l].lpSibling == BasicBlock::NOT_IN_LOOP) - { - lastChildLoop = l; - } - - JITDUMP("Resetting parent of loop number " FMT_LP " from " FMT_LP " to " FMT_LP ".\n", l, - optLoopTable[l].lpParent, loop.lpParent); - optLoopTable[l].lpParent = loop.lpParent; - } - - if (lastChildLoop != BasicBlock::NOT_IN_LOOP) - { - JITDUMP(FMT_LP " has child loops. Update sibling link " FMT_LP " -> " FMT_LP ".\n", loopNum, - lastChildLoop, nextSibling); - - optLoopTable[lastChildLoop].lpSibling = nextSibling; - } - else - { - assert(!"There is atleast one loop, but found none."); - } - - // Finally, convey that there are no children of `loopNum` - loop.lpChild = BasicBlock::NOT_IN_LOOP; - } - } - else - { - // If there are no top-level loops, then all the child loops, - // become the top-level loops. - for (BasicBlock::loopNumber l = loop.lpChild; // - l != BasicBlock::NOT_IN_LOOP; // - l = optLoopTable[l].lpSibling) - { - assert(!optLoopTable[l].lpIsRemoved()); - - JITDUMP("Marking loop number " FMT_LP " from " FMT_LP " as top level loop.\n", l, optLoopTable[l].lpParent); - optLoopTable[l].lpParent = BasicBlock::NOT_IN_LOOP; - } - } - - // Unmark any preheader - // - if ((loop.lpFlags & LPFLG_HAS_PREHEAD) != 0) - { - loop.lpHead->RemoveFlags(BBF_LOOP_PREHEADER); - } - - loop.lpFlags |= LPFLG_REMOVED; - -#ifdef DEBUG - if (optAnyChildNotRemoved(loopNum)) - { - JITDUMP("Removed loop " FMT_LP " has one or more live children\n", loopNum); - } - - if (verbose) - { - printf("Removed " FMT_LP "\n", loopNum); - optPrintLoopTable(); - } - -// Note: we can't call `fgDebugCheckLoopTable()` here because if there are live children, it will assert. -// Assume the caller is going to fix up the table and `bbNatLoopNum` block annotations before the next time -// `fgDebugCheckLoopTable()` is called. -#endif // DEBUG -} diff --git a/src/coreclr/jit/phase.cpp b/src/coreclr/jit/phase.cpp index 25371e203588a..717d0a7d270d5 100644 --- a/src/coreclr/jit/phase.cpp +++ b/src/coreclr/jit/phase.cpp @@ -158,7 +158,7 @@ void Phase::PostPhase(PhaseStatus status) if ((comp->activePhaseChecks & PhaseChecks::CHECK_LOOPS) == PhaseChecks::CHECK_LOOPS) { - comp->fgDebugCheckLoopTable(); + comp->fgDebugCheckLoops(); } if ((comp->activePhaseChecks & PhaseChecks::CHECK_PROFILE) == PhaseChecks::CHECK_PROFILE) diff --git a/src/coreclr/jit/redundantbranchopts.cpp b/src/coreclr/jit/redundantbranchopts.cpp index 235a5fc1d5f40..0201a62ddf7b7 100644 --- a/src/coreclr/jit/redundantbranchopts.cpp +++ b/src/coreclr/jit/redundantbranchopts.cpp @@ -1007,42 +1007,6 @@ bool Compiler::optJumpThreadCheck(BasicBlock* const block, BasicBlock* const dom return false; } - if (optLoopTableValid) - { - // If block is a loop header, skip jump threading. - // - // This is an artificial limitation to ensure that subsequent loop table valididity - // checking can pass. We do not expect a loop entry to have multiple non-loop predecessors. - // - // This only blocks jump threading in a small number of cases. - // Revisit once we can ensure that loop headers are not critical blocks. - // - // Likewise we can't properly update the loop table if we thread the entry block. - // Not clear how much impact this has. - // - for (unsigned loopNum = 0; loopNum < optLoopCount; loopNum++) - { - const LoopDsc& loop = optLoopTable[loopNum]; - - if (loop.lpIsRemoved()) - { - continue; - } - - if (block == loop.lpHead) - { - JITDUMP(FMT_BB " is the header for " FMT_LP "; no threading\n", block->bbNum, loopNum); - return false; - } - - if (block == loop.lpEntry) - { - JITDUMP(FMT_BB " is the entry for " FMT_LP "; no threading\n", block->bbNum, loopNum); - return false; - } - } - } - // Verify that dom block dominates all of block's predecessors. // // This will initially be true but if we jump thread through