diff --git a/ecmascript/compiler/codegen/maple/maple_be/include/be/becommon.h b/ecmascript/compiler/codegen/maple/maple_be/include/be/becommon.h index 0cb7fc6d8aed3f28a0842a997743b1e7cd7805b4..8e5fb7ff282c7b7a024c104d5435ee73f2758833 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/include/be/becommon.h +++ b/ecmascript/compiler/codegen/maple/maple_be/include/be/becommon.h @@ -30,6 +30,11 @@ using namespace maple; enum BitsPerByte : uint8 { kBitsPerByte = 8, kLog2BitsPerByte = 3 }; +inline uint32 GetPointerBitSize() +{ + return GetPointerSize() * kBitsPerByte; +} + class JClassFieldInfo { /* common java class field info */ public: /* constructors */ diff --git a/ecmascript/compiler/codegen/maple/maple_be/include/be/common_utils.h b/ecmascript/compiler/codegen/maple/maple_be/include/be/common_utils.h index 154a0569227c93c1c69c03c166d64b78aa14dec6..75f424baa893ea6f54e104d73160d04dbe16a5c7 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/include/be/common_utils.h +++ b/ecmascript/compiler/codegen/maple/maple_be/include/be/common_utils.h @@ -41,9 +41,11 @@ constexpr uint32 k5BitSize = 5; constexpr uint32 k6BitSize = 6; constexpr uint32 k7BitSize = 7; constexpr uint32 k8BitSize = 8; +constexpr uint32 k12BitSize = 12; constexpr uint32 k16BitSize = 16; constexpr uint32 k24BitSize = 24; constexpr uint32 k32BitSize = 32; +constexpr uint32 k40BitSize = 40; constexpr uint32 k48BitSize = 48; constexpr uint32 k56BitSize = 56; constexpr uint32 k64BitSize = 64; diff --git a/ecmascript/compiler/codegen/maple/maple_be/include/cg/aarch64/aarch64_cfgo.h b/ecmascript/compiler/codegen/maple/maple_be/include/cg/aarch64/aarch64_cfgo.h index 763d1202d9ec90fdb952d84740d5f252e982baba..436afaaba56ac2510056a9694d549d0acdf0aad6 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/include/cg/aarch64/aarch64_cfgo.h +++ b/ecmascript/compiler/codegen/maple/maple_be/include/cg/aarch64/aarch64_cfgo.h @@ -17,24 +17,39 @@ #define MAPLEBE_INCLUDE_CG_AARCH64_AARCH64_CFGO_H #include "cfgo.h" +#include "aarch64_isa.h" namespace maplebe { class AArch64CFGOptimizer : public CFGOptimizer { public: AArch64CFGOptimizer(CGFunc &func, MemPool &memPool) : CFGOptimizer(func, memPool) {} - ~AArch64CFGOptimizer() = default; + ~AArch64CFGOptimizer() override = default; void InitOptimizePatterns() override; }; class AArch64FlipBRPattern : public FlipBRPattern { public: explicit AArch64FlipBRPattern(CGFunc &func) : FlipBRPattern(func) {} - ~AArch64FlipBRPattern() = default; + ~AArch64FlipBRPattern() override = default; private: uint32 GetJumpTargetIdx(const Insn &insn) override; MOperator FlipConditionOp(MOperator flippedOp) override; }; + +class AArch64CrossJumpBBPattern : public CrossJumpBBPattern { +public: + explicit AArch64CrossJumpBBPattern(CGFunc &func) : CrossJumpBBPattern(func) {} + ~AArch64CrossJumpBBPattern() override = default; + +private: + uint32 GetJumpTargetIdx(const Insn &insn) const override; + MOperator FlipConditionOp(MOperator flippedOp) const override; + MOperator GetUnCondBranchMOP() const override + { + return MOP_xuncond; + } +}; } /* namespace maplebe */ #endif /* MAPLEBE_INCLUDE_CG_AARCH64_AARCH64_CFGO_H */ diff --git a/ecmascript/compiler/codegen/maple/maple_be/include/cg/aarch64/aarch64_isa.h b/ecmascript/compiler/codegen/maple/maple_be/include/cg/aarch64/aarch64_isa.h index 1bbf5e7111a785615436181909ddc352959013b4..aa86c0b17c3fe641ef4470ff5d85e2c01ff8158e 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/include/cg/aarch64/aarch64_isa.h +++ b/ecmascript/compiler/codegen/maple/maple_be/include/cg/aarch64/aarch64_isa.h @@ -163,6 +163,10 @@ static inline bool IsPseudoInstruction(MOperator mOp) uint32 GetJumpTargetIdx(const Insn &insn); MOperator FlipConditionOp(MOperator flippedOp); + +// Function: for immediate verification, memopnd ofstvalue is returned from opnd input. +// It's worth noting that 0 will be returned when kBOR memopnd is input. +int64 GetMemOpndOffsetValue(Operand *o); } /* namespace AArch64isa */ /* @@ -176,7 +180,7 @@ inline void GetNextOffsetCalleeSaved(int &offset) offset += (kIntregBytelen << 1); } -MOperator GetMopPair(MOperator mop); +MOperator GetMopPair(MOperator mop, bool isIncludeStrbStrh); } /* namespace maplebe */ #endif /* MAPLEBE_INCLUDE_CG_AARCH64_AARCH64_ISA_H */ diff --git a/ecmascript/compiler/codegen/maple/maple_be/include/cg/aarch64/aarch64_mem_reference.h b/ecmascript/compiler/codegen/maple/maple_be/include/cg/aarch64/aarch64_mem_reference.h new file mode 100644 index 0000000000000000000000000000000000000000..319b31247baa6d21c5c4d818626365ef89cd9a9d --- /dev/null +++ b/ecmascript/compiler/codegen/maple/maple_be/include/cg/aarch64/aarch64_mem_reference.h @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MAPLEBE_INCLUDE_CG_AARCH64_AARCH64_MEM_REFERENCE_H +#define MAPLEBE_INCLUDE_CG_AARCH64_AARCH64_MEM_REFERENCE_H + +#include "deps.h" +#include "aarch64_insn.h" + +namespace maplebe { +class AArch64MemReference { +public: + // If one of the memory is volatile, we need build dependency. + static bool HasVolatile(const Insn &memInsn1, const Insn &memInsn2) + { + auto *memOpnd1 = static_cast(memInsn1.GetMemOpnd()); + auto *memOpnd2 = static_cast(memInsn2.GetMemOpnd()); + if (memOpnd1 != nullptr && memOpnd2 != nullptr) { + if (memOpnd1->IsVolatile() || memOpnd2->IsVolatile()) { + return true; + } + } + return false; + } + + // If the two memory reference sets do not have the same element(ostIdx), + // there is no alias, return false. + static bool HasMemoryReferenceAlias(const MemDefUseSet &refSet1, const MemDefUseSet &refSet2) + { + // Process conservatively when no alias info is available + if (refSet1.empty() || refSet2.empty()) { + return true; + } + for (const auto &ostIdx : refSet2) { + if (refSet1.find(ostIdx) != refSet1.end()) { + return true; + } + } + return false; + } + + // Check memory overlap of same baseAddress in basic analysis and + // can be used for both [stack] and [heap] memory. + static bool IsMemoryOverlap(const Insn &memInsn1, const Insn &memInsn2, const Insn *baseDefInsn1, + const Insn *baseDefInsn2) + { + auto *memOpnd1 = static_cast(memInsn1.GetMemOpnd()); + auto *memOpnd2 = static_cast(memInsn2.GetMemOpnd()); + ASSERT_NOT_NULL(memOpnd1); + ASSERT_NOT_NULL(memOpnd2); + // Must be BOI-mode + DEBUG_ASSERT(memOpnd1->GetAddrMode() == MemOperand::kAddrModeBOi && + memOpnd2->GetAddrMode() == MemOperand::kAddrModeBOi, + "invalid addressing mode"); + + RegOperand *baseOpnd1 = memOpnd1->GetBaseRegister(); + RegOperand *baseOpnd2 = memOpnd2->GetBaseRegister(); + // BaseRegister will be nullptr in BOI-mode, in what cases? + if (baseOpnd1 == nullptr || baseOpnd2 == nullptr) { + return true; + } + + // If the two base addresses are different, we can not analyze whether they overlap here, + // so we process conservatively. + if (baseOpnd1->GetRegisterNumber() != baseOpnd2->GetRegisterNumber()) { + return true; + } + if (baseDefInsn1 != baseDefInsn2) { + return true; + } + + OfstOperand *ofstOpnd1 = memOpnd1->GetOffsetImmediate(); + OfstOperand *ofstOpnd2 = memOpnd2->GetOffsetImmediate(); + + int64 ofstValue1 = (ofstOpnd1 == nullptr ? 0 : ofstOpnd1->GetOffsetValue()); + int64 ofstValue2 = (ofstOpnd2 == nullptr ? 0 : ofstOpnd2->GetOffsetValue()); + + // Compatible with the load/store pair and load/store + uint32 memByteSize1 = memInsn1.GetMemoryByteSize(); + uint32 memByteSize2 = memInsn2.GetMemoryByteSize(); + + int64 memByteBoundary1 = ofstValue1 + static_cast(memByteSize1); + int64 memByteBoundary2 = ofstValue2 + static_cast(memByteSize2); + + // no overlap + // baseAddr ofst2 ofst1 ofst2---> + // |________|___memSize2_____|_____memSize1_____|__________ + // | | + // memBoundary2 memBoundary1 + if (ofstValue2 >= memByteBoundary1 || ofstValue1 >= memByteBoundary2) { + return false; + } + return true; + } + + // Simply distinguish irrelevant stack memory. + static bool HasBasicMemoryDep(const Insn &memInsn1, const Insn &memInsn2, Insn *baseDefInsn1, Insn *baseDefInsn2) + { + // The callee-save and callee-reload instructions do not conflict with any other memory access instructions, + // we use the $isIndependent field to check that. + const MemDefUse *memReference1 = memInsn1.GetReferenceOsts(); + const MemDefUse *memReference2 = memInsn2.GetReferenceOsts(); + if (memReference1 != nullptr && memReference1->IsIndependent()) { + return false; + } + if (memReference2 != nullptr && memReference2->IsIndependent()) { + return false; + } + + auto *memOpnd1 = static_cast(memInsn1.GetMemOpnd()); + auto *memOpnd2 = static_cast(memInsn2.GetMemOpnd()); + // Not here to consider StImmOperand(e.g. adrp) or call(e.g. bl) + if (memOpnd1 == nullptr || memOpnd2 == nullptr) { + return true; + } + // 1. Check MIRSymbol + if (memOpnd1->GetSymbol() != nullptr && memOpnd2->GetSymbol() != nullptr && + memOpnd1->GetSymbol() != memOpnd2->GetSymbol()) { + return false; + } + // Only consider the BOI-mode for basic overlap analysis + if (memOpnd1->GetAddrMode() != MemOperand::kAddrModeBOi || + memOpnd2->GetAddrMode() != MemOperand::kAddrModeBOi) { + return true; + } + RegOperand *baseOpnd1 = memOpnd1->GetBaseRegister(); + RegOperand *baseOpnd2 = memOpnd2->GetBaseRegister(); + // BaseRegister will be nullptr in BOI-mode, in what cases? + if (baseOpnd1 == nullptr || baseOpnd2 == nullptr) { + return true; + } + // 2. Check memory overlap + if (!IsMemoryOverlap(memInsn1, memInsn2, baseDefInsn1, baseDefInsn2)) { + return false; + } + return true; + } + + static bool HasAliasMemoryDep(const Insn &fromMemInsn, const Insn &toMemInsn, DepType depType) + { + const MemDefUse *fromReferenceOsts = fromMemInsn.GetReferenceOsts(); + const MemDefUse *toReferenceOsts = toMemInsn.GetReferenceOsts(); + // Process conservatively when no alias info is available + if (fromReferenceOsts == nullptr || toReferenceOsts == nullptr) { + return true; + } + const MemDefUseSet &fromMemDefSet = fromReferenceOsts->GetDefSet(); + const MemDefUseSet &fromMemUseSet = fromReferenceOsts->GetUseSet(); + const MemDefUseSet &toMemDefSet = toReferenceOsts->GetDefSet(); + const MemDefUseSet &toMemUseSet = toReferenceOsts->GetUseSet(); + + switch (depType) { + // Check write->read dependency + case kDependenceTypeTrue: + return HasMemoryReferenceAlias(fromMemDefSet, toMemUseSet); + // Check read->write dependency + case kDependenceTypeAnti: + return HasMemoryReferenceAlias(fromMemUseSet, toMemDefSet); + // Check write->write dependency + case kDependenceTypeOutput: + return HasMemoryReferenceAlias(fromMemDefSet, toMemDefSet); + case kDependenceTypeNone: { + if (HasMemoryReferenceAlias(fromMemDefSet, toMemDefSet) || + HasMemoryReferenceAlias(fromMemDefSet, toMemUseSet) || + HasMemoryReferenceAlias(fromMemUseSet, toMemDefSet)) { + return true; + } else { + return false; + } + } + default: + CHECK_FATAL(false, "invalid memory dependency type"); + } + } + + // The entrance: if memory dependency needs to be build, return true. + // fromMemInsn & toMemInsn: will be [load, store, adrp, call] + // baseDefInsns: for basic memory dependency analysis, they define the base address and can be nullptr. + // depType: for memory analysis based on me-alias-info, only consider TRUE, ANTI, OUTPUT and NONE dependency type + // from fromMemInsn to toMemInsn, if it is NONE, we take care of both def and use sets. + static bool NeedBuildMemoryDependency(const Insn &fromMemInsn, const Insn &toMemInsn, Insn *baseDefInsn1, + Insn *baseDefInsn2, DepType depType) + { + // 0. Check volatile + if (HasVolatile(fromMemInsn, toMemInsn)) { + return true; + } + // 1. Do basic analysis of memory-related instructions based on cg-ir + if (!HasBasicMemoryDep(fromMemInsn, toMemInsn, baseDefInsn1, baseDefInsn2)) { + return false; + } + // 2. Do alias analysis of memory-related instructions based on me-alias-info + return HasAliasMemoryDep(fromMemInsn, toMemInsn, depType); + } +}; +} // namespace maplebe + +#endif // MAPLEBE_INCLUDE_CG_AARCH64_AARCH64_MEM_REFERENCE_H diff --git a/ecmascript/compiler/codegen/maple/maple_be/include/cg/aarch64/aarch64_optimize_common.h b/ecmascript/compiler/codegen/maple/maple_be/include/cg/aarch64/aarch64_optimize_common.h index a08b6854a6bddc0c43ae34ee81fe78fecb65358c..6ff3b95c15ec3a64175d6612a22b8c30ef7dd806 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/include/cg/aarch64/aarch64_optimize_common.h +++ b/ecmascript/compiler/codegen/maple/maple_be/include/cg/aarch64/aarch64_optimize_common.h @@ -36,8 +36,15 @@ public: LabelIdx GetJumpLabel(const Insn &insn) const override; bool IsCompareInsn(const Insn &insn) const override; bool IsCompareAndBranchInsn(const Insn &insn) const override; + bool IsTestAndSetCCInsn(const Insn &insn) const override; + bool IsTestAndBranchInsn(const Insn &insn) const override; bool IsAddOrSubInsn(const Insn &insn) const override; + bool IsSimpleJumpInsn(const Insn &insn) const override; RegOperand *CreateVregFromReg(const RegOperand &pReg) override; + void ReTargetSuccBB(BB &bb, LabelIdx newTarget) const override; + void FlipIfBB(BB &bb, LabelIdx ftLabel) const override; + BB *CreateGotoBBAfterCondBB(BB &bb, BB &fallthru, bool isTargetFallthru) const override; + void ModifyFathruBBToGotoBB(BB &bb, LabelIdx labelIdx) const override; }; } /* namespace maplebe */ diff --git a/ecmascript/compiler/codegen/maple/maple_be/include/cg/aarch64/aarch64_peep.h b/ecmascript/compiler/codegen/maple/maple_be/include/cg/aarch64/aarch64_peep.h index 4d53a3ef7eff75af0725920bde338d5e95a145de..59c32faccb34071df7342f0e6420263d47ff98d0 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/include/cg/aarch64/aarch64_peep.h +++ b/ecmascript/compiler/codegen/maple/maple_be/include/cg/aarch64/aarch64_peep.h @@ -23,13 +23,13 @@ #include "mir_builder.h" namespace maplebe { -class AArch64CGPeepHole : CGPeepHole { +class AArch64CGPeepHole : public CGPeepHole { public: /* normal constructor */ AArch64CGPeepHole(CGFunc &f, MemPool *memPool) : CGPeepHole(f, memPool) {}; /* constructor for ssa */ AArch64CGPeepHole(CGFunc &f, MemPool *memPool, CGSSAInfo *cgssaInfo) : CGPeepHole(f, memPool, cgssaInfo) {}; - ~AArch64CGPeepHole() = default; + ~AArch64CGPeepHole() override = default; void Run() override; bool DoSSAOptimize(BB &bb, Insn &insn) override; @@ -69,21 +69,33 @@ private: bool reverse = false; }; -/* - * Example 1) - * mov w5, #1 +/* Condition: mov_imm1 value is 0(1/-1) && mov_imm value is 1/-1(0) + * + * Pattern 1: Two mov insn + One csel insn + * + * Example 1: + * mov w5, #1 * ... - * mov w0, #0 - * csel w5, w5, w0, NE ===> cset w5, NE + * mov w0, #0 + * csel w5, w5, w0, NE ===> cset w5, NE * - * Example 2) - * mov w5, #0 + * Example 2: + * mov w5, #0 * ... - * mov w0, #1 - * csel w5, w5, w0, NE ===> cset w5,EQ + * mov w0, #-1 + * csel w5, w5, w0, NE ===> csetm w5, EQ + * + * Pattern 2: One mov insn + One csel insn with RZR + * + * Example 1: + * mov w0, #4294967295 + * ...... ====> csetm w1, EQ + * csel w1, w0, wzr, EQ * - * conditions: - * 1. mov_imm1 value is 0(1) && mov_imm value is 1(0) + * Example 2: + * mov w0, #1 + * ...... ====> cset w1, LE + * csel w1, wzr, w0, GT */ class CselToCsetPattern : public CGPeepPattern { public: @@ -91,6 +103,7 @@ public: : CGPeepPattern(cgFunc, currBB, currInsn, info) { } + CselToCsetPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn) : CGPeepPattern(cgFunc, currBB, currInsn) {} ~CselToCsetPattern() override = default; void Run(BB &bb, Insn &insn) override; bool CheckCondition(Insn &insn) override; @@ -102,8 +115,91 @@ public: private: bool IsOpndDefByZero(const Insn &insn) const; bool IsOpndDefByOne(const Insn &insn) const; + bool IsOpndDefByAllOnes(const Insn &insn) const; + bool CheckZeroCondition(const Insn &insn); + Insn *BuildCondSetInsn(const Insn &cselInsn) const; + void ZeroRun(BB &bb, Insn &insn); + bool isOne = false; + bool isAllOnes = false; + bool isZeroBefore = false; + Insn *prevMovInsn = nullptr; Insn *prevMovInsn1 = nullptr; Insn *prevMovInsn2 = nullptr; + RegOperand *useReg = nullptr; +}; + +// cmp w2, w3/imm +// csel w0, w1, w1, NE ===> mov w0, w1 +class CselToMovPattern : public CGPeepPattern { +public: + CselToMovPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn) : CGPeepPattern(cgFunc, currBB, currInsn) {} + ~CselToMovPattern() override = default; + void Run(BB &bb, Insn &insn) override; + bool CheckCondition(Insn &insn) override; + std::string GetPatternName() override + { + return "CselToMovPattern"; + } +}; + +/* + * mov w1, #1 + * csel w2, w1, w0, EQ ===> csinc w2, w0, WZR, NE + * + * mov w1, #1 + * csel w2, w0, w1, EQ ===> csinc w2, w0, WZR, EQ + */ +class CselToCsincRemoveMovPattern : public CGPeepPattern { +public: + CselToCsincRemoveMovPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn, CGSSAInfo &info) + : CGPeepPattern(cgFunc, currBB, currInsn, info) + { + } + ~CselToCsincRemoveMovPattern() override = default; + void Run(BB &bb, Insn &insn) override; + bool CheckCondition(Insn &insn) override; + std::string GetPatternName() override + { + return "CselToCsincRemoveMovPattern"; + } + +private: + bool IsOpndMovOneAndNewOpndOpt(const Insn &curInsn); + Insn *prevMovInsn = nullptr; + RegOperand *newSecondOpnd = nullptr; + CondOperand *cond = nullptr; + bool needReverseCond = false; +}; + +/* + * cset w0, HS + * add w2, w1, w0 ===> cinc w2, w1, hs + * + * cset w0, HS + * add w2, w0, w1 ===> cinc w2, w1, hs + */ +class CsetToCincPattern : public CGPeepPattern { +public: + CsetToCincPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn, CGSSAInfo &info) + : CGPeepPattern(cgFunc, currBB, currInsn, info) + { + } + ~CsetToCincPattern() override + { + defInsn = nullptr; + } + void Run(BB &bb, Insn &insn) override; + bool CheckCondition(Insn &insn) override; + bool CheckDefInsn(const RegOperand &opnd, Insn &insn); + bool CheckRegTyCc(const Insn &tempDefInsn, Insn &insn) const; + std::string GetPatternName() override + { + return "CsetToCincPattern"; + } + +private: + Insn *defInsn = nullptr; + int32 csetOpnd1 = 0; }; /* @@ -139,6 +235,7 @@ public: private: MOperator SelectNewMop(ConditionCode condCode, bool inverse) const; Insn *prevInsn = nullptr; + LabelOperand *labelOpnd = nullptr; }; /* @@ -174,6 +271,79 @@ private: Insn *prevInsn = nullptr; }; +/* + * case: + * ldr R261(32), [R197, #300] + * ldr R262(32), [R208, #12] + * cmp (CC) R261, R262 + * bne lable175. + * ldr R264(32), [R197, #304] + * ldr R265(32), [R208, #16] + * cmp (CC) R264, R265 + * bne lable175. + * ====> + * ldr R261(64), [R197, #300] + * ldr R262(64), [R208, #12] + * cmp (CC) R261, R262 + * bne lable175. + */ +class LdrCmpPattern : public CGPeepPattern { +public: + LdrCmpPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn) : CGPeepPattern(cgFunc, currBB, currInsn) {} + ~LdrCmpPattern() override + { + prevLdr1 = nullptr; + prevLdr2 = nullptr; + ldr1 = nullptr; + ldr2 = nullptr; + prevCmp = nullptr; + bne1 = nullptr; + bne2 = nullptr; + } + void Run(BB &bb, Insn &insn) override; + bool CheckCondition(Insn &insn) override; + std::string GetPatternName() override + { + return "LdrCmpPattern"; + } + +private: + bool IsLdr(const Insn *insn) const + { + if (insn == nullptr) { + return false; + } + return insn->GetMachineOpcode() == MOP_wldr; + } + + bool IsCmp(const Insn *insn) const + { + if (insn == nullptr) { + return false; + } + return insn->GetMachineOpcode() == MOP_wcmprr; + } + + bool IsBne(const Insn *insn) const + { + if (insn == nullptr) { + return false; + } + return insn->GetMachineOpcode() == MOP_bne; + } + + bool SetInsns(); + bool CheckInsns() const; + bool MemOffet4Bit(const MemOperand &m1, const MemOperand &m2) const; + Insn *prevLdr1 = nullptr; + Insn *prevLdr2 = nullptr; + Insn *ldr1 = nullptr; + Insn *ldr2 = nullptr; + Insn *prevCmp = nullptr; + Insn *bne1 = nullptr; + Insn *bne2 = nullptr; +}; + /* * combine {sxtw / uxtw} & lsl ---> {sbfiz / ubfiz} * sxtw x1, w0 @@ -326,6 +496,28 @@ private: bool op2IsMvnDef = false; }; +// redundancy and elimination +// and w9, w8, #65535 +// rev16 w10, w9 rev16 w10, w8 +// strh w10, [x0,#10] ===> strh w10, [x0,#10] +class DeleteAndBeforeRevStrPattern : public CGPeepPattern { +public: + DeleteAndBeforeRevStrPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn, CGSSAInfo &info) + : CGPeepPattern(cgFunc, currBB, currInsn, info) + { + } + DeleteAndBeforeRevStrPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn) : CGPeepPattern(cgFunc, currBB, currInsn) + { + } + ~DeleteAndBeforeRevStrPattern() override {} + void Run(BB &bb, Insn &insn) override; + bool CheckCondition(Insn &insn) override; + std::string GetPatternName() override + { + return "DeleteAndBeforeRevStrPattern"; + } +}; + /* * and r0, r1, #4 (the imm is n power of 2) * ... @@ -356,6 +548,102 @@ private: Insn *prevInsn = nullptr; }; +/* and r0, r1, #1 + * ... + * tbz r0, #0, .label + * ===> tbz r1, #0 .label + */ +class AndTbzPattern : public CGPeepPattern { +public: + AndTbzPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn, CGSSAInfo &info) + : CGPeepPattern(cgFunc, currBB, currInsn, info) + { + } + AndTbzPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn) : CGPeepPattern(cgFunc, currBB, currInsn) {} + ~AndTbzPattern() override + { + prevInsn = nullptr; + } + void Run(BB &bb, Insn &insn) override; + bool CheckCondition(Insn &insn) override; + std::string GetPatternName() override + { + return "AndTbzPattern"; + } + +private: + Insn *prevInsn = nullptr; +}; + +/* Norm pattern + * combine {rev / rev16} & {tbz / tbnz} ---> {tbz / tbnz} + * rev16 w0, w0 + * tbz w0, #14 ===> tbz w0, #6 + * + * rev w0, w0 + * tbz w0, #14 ===> tbz w0, #22 + */ +class NormRevTbzToTbzPattern : public CGPeepPattern { +public: + NormRevTbzToTbzPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn, CGSSAInfo &info) + : CGPeepPattern(cgFunc, currBB, currInsn, info) + { + } + NormRevTbzToTbzPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn) : CGPeepPattern(cgFunc, currBB, currInsn) {} + ~NormRevTbzToTbzPattern() override + { + tbzInsn = nullptr; + } + std::string GetPatternName() override + { + return "NormRevTbzToTbzPattern"; + } + bool CheckCondition(Insn &insn) override; + void Run(BB &bb, Insn &insn) override; + +private: + void SetRev16Value(const uint32 &oldValue, uint32 &revValue) const; + void SetWrevValue(const uint32 &oldValue, uint32 &revValue) const; + void SetXrevValue(const uint32 &oldValue, uint32 &revValue) const; + Insn *tbzInsn = nullptr; +}; + +/* Add/Sub & load/store insn mergence pattern: + * add x0, x0, #255 + * ldr w1, [x0] ====> ldr w1, [x0, #255]! + * + * stp w1, w2, [x0] + * sub x0, x0, #256 ====> stp w1, w2, [x0], #-256 + * If new load/store insn is invalid and should be split, the pattern optimization will not work. + */ +class AddSubMergeLdStPattern : public CGPeepPattern { +public: + AddSubMergeLdStPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn, CGSSAInfo &info) + : CGPeepPattern(cgFunc, currBB, currInsn, info) + { + } + AddSubMergeLdStPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn) : CGPeepPattern(cgFunc, currBB, currInsn) {} + ~AddSubMergeLdStPattern() override = default; + std::string GetPatternName() override + { + return "AddSubMergeLdStPattern"; + } + bool CheckCondition(Insn &insn) override; + void Run(BB &bb, Insn &insn) override; + +private: + bool CheckIfCanBeMerged(const Insn *adjacentInsn, const Insn &insn); + Insn *FindRegInBB(const Insn &insn, bool isAbove) const; + Insn *nextInsn = nullptr; + Insn *prevInsn = nullptr; + Insn *insnToBeReplaced = nullptr; + bool isAddSubFront = false; + bool isLdStFront = false; + bool isInsnAdd = false; + RegOperand *insnDefReg = nullptr; + RegOperand *insnUseReg = nullptr; +}; + class CombineSameArithmeticPattern : public CGPeepPattern { public: CombineSameArithmeticPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn, CGSSAInfo &info) @@ -484,7 +772,7 @@ public: } protected: - enum SpecificExtType : uint8 { EXTUNDEF = 0, SXTB, SXTH, SXTW, UXTB, UXTH, UXTW, SpecificExtTypeSize }; + enum SpecificExtType : uint8 { EXTUNDEF = 0, SXTB, SXTH, SXTW, UXTB, UXTH, UXTW, AND, EXTTYPESIZE }; enum OptSceneType : uint8 { kSceneUndef = 0, kSceneMov, kSceneLoad, kSceneSameExt }; static constexpr uint8 kPrevLoadPatternNum = 6; static constexpr uint8 kPrevLoadMappingNum = 2; @@ -492,7 +780,7 @@ protected: static constexpr uint64 kInvalidValue = 0; static constexpr uint8 kSameExtPatternNum = 4; static constexpr uint8 kSameExtMappingNum = 2; - uint64 extValueRangeTable[SpecificExtTypeSize][kValueTypeNum] = { + uint64 extValueRangeTable[EXTTYPESIZE][kValueTypeNum] = { /* {minValue, maxValue} */ {kInvalidValue, kInvalidValue}, /* UNDEF */ {0xFFFFFFFFFFFFFF80, 0x7F}, /* SXTB */ @@ -500,9 +788,10 @@ protected: {0xFFFFFFFF80000000, kInvalidValue}, /* SXTW */ {0xFFFFFFFFFFFFFF00, kInvalidValue}, /* UXTB */ {0xFFFFFFFFFFFF0000, kInvalidValue}, /* UXTH */ - {kInvalidValue, kInvalidValue} /* UXTW */ + {kInvalidValue, kInvalidValue}, /* UXTW */ + {kInvalidValue, kInvalidValue}, /* AND */ }; - MOperator loadMappingTable[SpecificExtTypeSize][kPrevLoadPatternNum][kPrevLoadMappingNum] = { + MOperator loadMappingTable[EXTTYPESIZE][kPrevLoadPatternNum][kPrevLoadMappingNum] = { /* {prevOrigMop, prevNewMop} */ {{MOP_undef, MOP_undef}, {MOP_undef, MOP_undef}, @@ -537,7 +826,7 @@ protected: {{MOP_wldrh, MOP_wldrh}, {MOP_wldrb, MOP_wldrb}, {MOP_wldr, MOP_wldrh}, - {MOP_undef, MOP_undef}, + {MOP_wldrsh, MOP_wldrh}, {MOP_undef, MOP_undef}, {MOP_undef, MOP_undef}}, /* UXTH */ {{MOP_wldr, MOP_wldr}, @@ -545,9 +834,15 @@ protected: {MOP_wldrb, MOP_wldrb}, {MOP_undef, MOP_undef}, {MOP_undef, MOP_undef}, - {MOP_undef, MOP_undef}} /* UXTW */ - }; - MOperator sameExtMappingTable[SpecificExtTypeSize][kSameExtPatternNum][kSameExtMappingNum] = { + {MOP_undef, MOP_undef}}, /* UXTW */ + {{MOP_wldrb, MOP_wldrb}, + {MOP_wldrsh, MOP_wldrb}, + {MOP_wldrh, MOP_wldrb}, + {MOP_xldrsw, MOP_wldrb}, + {MOP_wldr, MOP_wldrb}, + {MOP_undef, MOP_undef}, + }}; + MOperator sameExtMappingTable[EXTTYPESIZE][kSameExtPatternNum][kSameExtMappingNum] = { /* {prevMop, currMop} */ {{MOP_undef, MOP_undef}, {MOP_undef, MOP_undef}, {MOP_undef, MOP_undef}, {MOP_undef, MOP_undef}}, /* UNDEF */ {{MOP_xsxtb32, MOP_xsxtb32}, @@ -566,18 +861,19 @@ protected: {{MOP_xuxtb32, MOP_xuxth32}, {MOP_xuxth32, MOP_xuxth32}, {MOP_undef, MOP_undef}, - {MOP_undef, MOP_undef}}, /* UXTH */ - {{MOP_xuxtw64, MOP_xuxtw64}, {MOP_undef, MOP_undef}, {MOP_undef, MOP_undef}, {MOP_undef, MOP_undef}} /* UXTW */ + {MOP_undef, MOP_undef}}, /* UXTH */ + {{MOP_xuxtw64, MOP_xuxtw64}, {MOP_undef, MOP_undef}, {MOP_undef, MOP_undef}, {MOP_undef, MOP_undef}}, /* UXTW */ + {{MOP_undef, MOP_undef}, {MOP_undef, MOP_undef}, {MOP_undef, MOP_undef}, {MOP_undef, MOP_undef}}, /* AND */ }; private: void SetSpecificExtType(const Insn &currInsn); void SetOptSceneType(); - bool IsValidLoadExtPattern(Insn &currInsn, MOperator oldMop, MOperator newMop) const; + bool IsValidLoadExtPattern(MOperator oldMop, MOperator newMop) const; MOperator SelectNewLoadMopByBitSize(MOperator lowBitMop) const; - void ElimExtensionAfterLoad(Insn &currInsn); - void ElimExtensionAfterMov(Insn &currInsn); - void ElimExtensionAfterSameExt(Insn &currInsn); + void ElimExtensionAfterLoad(Insn &insn); + void ElimExtensionAfterMov(Insn &insn); + void ElimExtensionAfterSameExt(Insn &insn); void ReplaceExtWithMov(Insn &currInsn); Insn *prevInsn = nullptr; SpecificExtType extTypeIdx = EXTUNDEF; @@ -711,8 +1007,8 @@ public: protected: enum ArithmeticType : uint8 { kUndef = 0, kAdd, kFAdd, kSub, kFSub, kNeg, kFNeg, kArithmeticTypeSize }; - static constexpr uint8 newMopNum = 2; - MOperator curMop2NewMopTable[kArithmeticTypeSize][newMopNum] = { + static constexpr uint8 kNewMopNum = 2; + MOperator curMop2NewMopTable[kArithmeticTypeSize][kNewMopNum] = { /* {32bit_mop, 64bit_mop} */ {MOP_undef, MOP_undef}, /* kUndef */ {MOP_wmaddrrrr, MOP_xmaddrrrr}, /* kAdd */ @@ -752,6 +1048,7 @@ public: ~LsrAndToUbfxPattern() override = default; void Run(BB &bb, Insn &insn) override; bool CheckCondition(Insn &insn) override; + bool CheckIntersectedCondition(const Insn &insn); std::string GetPatternName() override { return "LsrAndToUbfxPattern"; @@ -759,6 +1056,38 @@ public: private: Insn *prevInsn = nullptr; + bool isWXSumOutOfRange = false; +}; + +/* + * lsl w1, w2, #m + * and w3, w1, #[(2^n-1 << m) ~ (2^n-1)] ---> if n > m : ubfiz w3, w2, #m, #n-m + * + * and w1, w2, #2^n-1 ---> ubfiz w3, w2, #m, #n + * lsl w3, w1, #m + * Exclude the scenarios that can be optimized by prop. + */ +class LslAndToUbfizPattern : public CGPeepPattern { +public: + LslAndToUbfizPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn, CGSSAInfo &info) + : CGPeepPattern(cgFunc, currBB, currInsn, info) + { + } + ~LslAndToUbfizPattern() override + { + defInsn = nullptr; + } + void Run(BB &bb, Insn &insn) override; + bool CheckCondition(Insn &insn) override; + Insn *BuildNewInsn(const Insn &andInsn, const Insn &lslInsn, const Insn &useInsn) const; + bool CheckUseInsnMop(const Insn &useInsn) const; + std::string GetPatternName() override + { + return "LslAndToUbfizPattern"; + } + +private: + Insn *defInsn = nullptr; }; /* @@ -823,6 +1152,7 @@ public: : CGPeepPattern(cgFunc, currBB, currInsn, info) { } + UbfxAndCbzToTbzPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn) : CGPeepPattern(cgFunc, currBB, currInsn) {} ~UbfxAndCbzToTbzPattern() override { useInsn = nullptr; @@ -839,17 +1169,51 @@ private: MOperator newMop = MOP_undef; }; -/* - * Looking for identical mem insn to eliminate. - * If two back-to-back is: - * 1. str + str - * 2. str + ldr - * And the [MEM] is pattern of [base + offset] - * 1. The [MEM] operand is exactly same then first - * str can be eliminate. - * 2. The [MEM] operand is exactly same and src opnd - * of str is same as the dest opnd of ldr then - * ldr can be eliminate +/* ubfx R1 R0 a b + * ubfx R2 R1 c d => ubfx R2 R0 a + c, min(b -c, d) + * --------------------------------------------------- + * for example: + * ubfx R1 R0 3 5 + * ubfx R2 R1 2 8 => ubfx R2 R0 5 (2+3), 3 (min(3, 8)) + */ +class UbfxAndMergetPattern : public CGPeepPattern { +public: + UbfxAndMergetPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn, CGSSAInfo &info) + : CGPeepPattern(cgFunc, currBB, currInsn, info) + { + } + UbfxAndMergetPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn) : CGPeepPattern(cgFunc, currBB, currInsn) {} + ~UbfxAndMergetPattern() override + { + prevSrc = nullptr; + } + void Run(BB &bb, Insn &insn) override; + bool CheckCondition(Insn &insn) override; + std::string GetPatternName() override + { + return "UbfxAndMergetPattern"; + } + +private: + bool IsAllOneToMSB(int64 val) const; + int32 GetMSB(int64 val) const; + int64 prevLsb = -1; + int64 prevWidth = -1; + int64 currLsb = -1; + int64 currWidth = -1; + MOperator newMop = MOP_undef; + RegOperand *prevSrc = nullptr; +}; + +/* Find up identical two mem insns in local bb to eliminate redundancy, as following: + * 1. str[BOI] + str[BOI] : + * Remove first str insn when the [MEM] operand is exactly same, and the [srcOpnd] of two str don't need to be same, + * and there is no redefinition of [srcOpnd] between two strs. + * 2. str[BOI] + ldr[BOI] : + * Remove ldr insn when the [MEM] operand is exactly same and the [srcOpnd] of str + * is same as the [destOpnd] of ldr, and there is no redefinition of [srcOpnd] and [destOpnd] between two insns. + * 3. ldr[BOI] + ldr[BOI] : + * Remove second ldr insn */ class RemoveIdenticalLoadAndStorePattern : public CGPeepPattern { public: @@ -859,7 +1223,7 @@ public: } ~RemoveIdenticalLoadAndStorePattern() override { - nextInsn = nullptr; + prevIdenticalInsn = nullptr; } void Run(BB &bb, Insn &insn) override; bool CheckCondition(Insn &insn) override; @@ -869,8 +1233,11 @@ public: } private: - bool IsMemOperandsIdentical(const Insn &insn1, const Insn &insn2) const; - Insn *nextInsn = nullptr; + bool IsIdenticalMemOpcode(const Insn &curInsn, const Insn &checkedInsn) const; + Insn *FindPrevIdenticalMemInsn(const Insn &curInsn) const; + bool HasImplictSizeUse(const Insn &curInsn) const; + bool HasMemReferenceBetweenTwoInsns(const Insn &curInsn) const; + Insn *prevIdenticalInsn = nullptr; }; /* ======== CGPeepPattern End ======== */ @@ -920,8 +1287,38 @@ public: }; /* - * Combining 2 STRs into 1 stp or 2 LDRs into 1 ldp, when they are - * back to back and the [MEM] they access is conjointed. + * mov dest1, imm + * mul dest2, reg1, dest1 + * ===> if imm is 2^n + * mov dest1, imm + * lsl dest2, reg1, n + */ +class MulImmToShiftPattern : public CGPeepPattern { +public: + MulImmToShiftPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn, CGSSAInfo &info) + : CGPeepPattern(cgFunc, currBB, currInsn, info) + { + } + ~MulImmToShiftPattern() override + { + movInsn = nullptr; + } + std::string GetPatternName() override + { + return "MulImmToShiftPattern"; + } + bool CheckCondition(Insn &insn) override; + void Run(BB &bb, Insn &insn) override; + +private: + Insn *movInsn = nullptr; + uint32 shiftVal = 0; + MOperator newMop = MOP_undef; +}; + +/* + * Combining {2 str into 1 stp || 2 ldr into 1 ldp || 2 strb into 1 strh || 2 strh into 1 str}, + * when they are back to back and the [MEM] they access is conjoined. */ class CombineContiLoadAndStorePattern : public CGPeepPattern { public: @@ -930,10 +1327,7 @@ public: { doAggressiveCombine = cgFunc.GetMirModule().IsCModule(); } - ~CombineContiLoadAndStorePattern() override - { - memOpnd = nullptr; - } + ~CombineContiLoadAndStorePattern() override = default; void Run(BB &bb, Insn &insn) override; bool CheckCondition(Insn &insn) override; std::string GetPatternName() override @@ -942,7 +1336,7 @@ public: } private: - std::vector FindPrevStrLdr(Insn &insn, regno_t destRegNO, regno_t memBaseRegNO, int64 baseOfst); + std::vector FindPrevStrLdr(Insn &insn, regno_t destRegNO, regno_t memBaseRegNO, int64 baseOfst) const; /* * avoid the following situation: * str x2, [x19, #8] @@ -950,30 +1344,28 @@ private: * bl foo (change memory) * str x21, [x19, #16] */ - bool IsRegNotSameMemUseInInsn(const Insn &insn, regno_t regNO, bool isStore, int64 baseOfst) const; + bool IsRegNotSameMemUseInInsn(const Insn &checkInsn, const Insn &curInsn, regno_t curBaseRegNO, bool isCurStore, + int64 curBaseOfst, int64 curMemRange) const; + bool IsValidNormalLoadOrStorePattern(const Insn &insn, const Insn &prevInsn, const MemOperand &memOpnd, + int64 curOfstVal, int64 prevOfstVal); + bool IsValidStackArgLoadOrStorePattern(const Insn &curInsn, const Insn &prevInsn, const MemOperand &curMemOpnd, + const MemOperand &prevMemOpnd, int64 curOfstVal, int64 prevOfstVal) const; + Insn *GenerateMemPairInsn(MOperator newMop, RegOperand &curDestOpnd, RegOperand &prevDestOpnd, + MemOperand &combineMemOpnd, bool isCurDestFirst); + bool FindUseX16AfterInsn(const Insn &curInsn) const; void RemoveInsnAndKeepComment(BB &bb, Insn &insn, Insn &prevInsn) const; - MOperator GetMopHigherByte(MOperator mop) const; - bool SplitOfstWithAddToCombine(const Insn &curInsn, Insn &combineInsn, const MemOperand &memOperand) const; - Insn *FindValidSplitAddInsn(Insn &curInsn, RegOperand &baseOpnd) const; - bool PlaceSplitAddInsn(const Insn &curInsn, Insn &combineInsn, const MemOperand &memOpnd, RegOperand &baseOpnd, - uint32 bitLen) const; + bool doAggressiveCombine = false; - MemOperand *memOpnd = nullptr; + bool isPairAfterCombine = true; }; /* - * mov x0, x1 - * ldr x0, [x0] --> ldr x0, [x1] + * add xt, xn, #imm add xt, xn, xm + * ldr xd, [xt] ldr xd, [xt] + * =====================> + * ldr xd, [xn, #imm] ldr xd, [xn, xm] * - * add x0, x0, #64 - * ldr x0, [x0] --> ldr x0, [x0, #64] - * - * add x0, x7, x0 - * ldr x11, [x0] --> ldr x11, [x0, x7] - * - * lsl x1, x4, #3 - * add x0, x0, x1 - * ldr d1, [x0] --> ldr d1, [x0, x4, lsl #3] + * load/store can do extend shift as well */ class EnhanceStrLdrAArch64 : public PeepPattern { public: @@ -1001,6 +1393,24 @@ public: void Run(BB &bb, Insn &insn) override; }; +class EliminateSpecifcSXTPattern : public CGPeepPattern { +public: + EliminateSpecifcSXTPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn) : CGPeepPattern(cgFunc, currBB, currInsn) {} + ~EliminateSpecifcSXTPattern() override + { + prevInsn = nullptr; + } + void Run(BB &bb, Insn &insn) override; + bool CheckCondition(Insn &insn) override; + std::string GetPatternName() override + { + return "EliminateSpecifcSXTPattern"; + } + +private: + Insn *prevInsn = nullptr; +}; + /* Eliminate the uxt[b|h|w] w0, w0;when w0 is satisify following: * i) mov w0, #imm (#imm is not out of range) * ii) mov w0, R0(Is return value of call and return size is not of range) @@ -1013,6 +1423,24 @@ public: void Run(BB &bb, Insn &insn) override; }; +class EliminateSpecifcUXTPattern : public CGPeepPattern { +public: + EliminateSpecifcUXTPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn) : CGPeepPattern(cgFunc, currBB, currInsn) {} + ~EliminateSpecifcUXTPattern() override + { + prevInsn = nullptr; + } + void Run(BB &bb, Insn &insn) override; + bool CheckCondition(Insn &insn) override; + std::string GetPatternName() override + { + return "EliminateSpecifcUXTPattern"; + } + +private: + Insn *prevInsn = nullptr; +}; + /* fmov ireg1 <- freg1 previous insn * fmov ireg2 <- freg1 current insn * use ireg2 may or may not be present @@ -1131,6 +1559,7 @@ public: prevInsn = nullptr; } void Run(BB &bb, Insn &insn) override; + bool HasImplicitSizeUse(const Insn &insn) const; bool CheckCondition(Insn &insn) override; std::string GetPatternName() override { @@ -1266,6 +1695,18 @@ public: void Run(BB &bb, Insn &insn) override; }; +class AndCbzBranchesToTstPattern : public CGPeepPattern { +public: + AndCbzBranchesToTstPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn) : CGPeepPattern(cgFunc, currBB, currInsn) {} + ~AndCbzBranchesToTstPattern() override = default; + void Run(BB &bb, Insn &insn) override; + bool CheckCondition(Insn &insn) override; + std::string GetPatternName() override + { + return "AndCbzBranchesToTstPattern"; + } +}; + /* * Optimize the following patterns: * and w0, w0, #1 ====> and w0, w0, #1 @@ -1314,6 +1755,29 @@ public: private: Insn *FindPreviousCmp(Insn &insn) const; }; + +class AndCmpBranchesToCsetPattern : public CGPeepPattern { +public: + AndCmpBranchesToCsetPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn, CGSSAInfo &info) + : CGPeepPattern(cgFunc, currBB, currInsn, info) + { + } + ~AndCmpBranchesToCsetPattern() override + { + prevCmpInsn = nullptr; + } + void Run(BB &bb, Insn &insn) override; + bool CheckCondition(Insn &insn) override; + std::string GetPatternName() override + { + return "AndCmpBranchesToCsetPattern"; + } + +private: + Insn *prevAndInsn = nullptr; + Insn *prevCmpInsn = nullptr; +}; + /* * We optimize the following pattern in this function: * cmp w[0-9]*, wzr ====> tbz w[0-9]*, #31, .label @@ -1341,6 +1805,43 @@ public: void Run(BB &bb, Insn &insn) override; }; +/* + * and x0, x0, #281474976710655 ====> eor x0, x0, x1 + * and x1, x1, #281474976710655 tst x0, 281474976710655 + * cmp x0, x1 bne .L.5187__150 + * bne .L.5187__150 + */ +class AndAndCmpBranchesToTstPattern : public CGPeepPattern { +public: + AndAndCmpBranchesToTstPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn, CGSSAInfo &info) + : CGPeepPattern(cgFunc, currBB, currInsn, info) + { + } + ~AndAndCmpBranchesToTstPattern() override + { + prevCmpInsn = nullptr; + } + + void Run(BB &bb, Insn &insn) override; + bool CheckCondition(Insn &insn) override; + std::string GetPatternName() override + { + return "AndAndCmpBranchesToCsetPattern"; + } + +private: + bool CheckCondInsn(const Insn &insn); + Insn *CheckAndGetPrevAndDefInsn(const RegOperand ®Opnd) const; + bool CheckAndSelectPattern(); + RegOperand *ccReg = nullptr; + Insn *prevPrevAndInsn = nullptr; + Insn *prevAndInsn = nullptr; + Insn *prevCmpInsn = nullptr; + MOperator newTstMop = MOP_undef; + MOperator newEorMop = MOP_undef; + int64 tstImmVal = -1; +}; + /* * Look for duplicate or overlapping zero or sign extensions. * Examples: @@ -1439,13 +1940,69 @@ public: void Run(BB &bb, Insn &insn) override; private: - bool PredBBCheck(BB &bb, bool checkCbz, const Operand &opnd) const; + bool PredBBCheck(BB &bb, bool checkCbz, const Operand &opnd, bool is64BitOnly) const; bool OpndDefByMovZero(const Insn &insn) const; bool NoPreDefine(Insn &testInsn) const; void ProcessBBHandle(BB *processBB, const BB &bb, const Insn &insn) const; + bool NoMoreThan32BitUse(Insn &testInsn) const; CGCFG *cgcfg; }; +/* we optimize the following scenarios in this pattern: + * for example 1: + * mov w1, #9 + * cmp w0, #1 => cmp w0, #1 + * mov w2, #8 csel w0, w0, wzr, EQ + * csel w0, w1, w2, EQ add w0, w0, #8 + * for example 2: + * mov w1, #8 + * cmp w0, #1 => cmp w0, #1 + * mov w2, #9 cset w0, NE + * csel w0, w1, w2, EQ add w0, w0, #8 + * for example 3: + * mov w1, #3 + * cmp w0, #4 => cmp w0, #4 + * mov w2, #7 csel w0, w0, wzr, EQ + * csel w0, w1, w2, NE add w0, w0, #3 + * condition: + * 1. The source operand of the two mov instructions are immediate operand; + * 2. The difference value between two immediates is equal to the value being compared in the cmp insn; + * 3. The reg w1 and w2 are not used in the instructions after csel; + * 4. The condOpnd in csel insn must be CC_NE or CC_EQ; + * 5. If the value in w1 is less than value in w2, condition in csel must be CC_NE, otherwise, + * the difference between them must be one; + * 6. If the value in w1 is more than value in w2, condition in csel must be CC_EQ, otherwise, + * the difference between them must be one. + */ +class CombineMovInsnBeforeCSelPattern : public CGPeepPattern { +public: + CombineMovInsnBeforeCSelPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn) + : CGPeepPattern(cgFunc, currBB, currInsn) + { + } + ~CombineMovInsnBeforeCSelPattern() override + { + insnMov2 = nullptr; + insnMov1 = nullptr; + cmpInsn = nullptr; + } + void Run(BB &bb, Insn &insn) override; + bool CheckCondition(Insn &insn) override; + std::string GetPatternName() override + { + return "CombineMovInsnBeforeCSelPattern"; + } + +private: + Insn *FindPrevMovInsn(const Insn &insn, regno_t regNo) const; + Insn *FindPrevCmpInsn(const Insn &insn) const; + Insn *insnMov2 = nullptr; + Insn *insnMov1 = nullptr; + Insn *cmpInsn = nullptr; + bool needReverseCond = false; + bool needCsetInsn = false; +}; + /* * We optimize the following pattern in this function: * if w0's valid bits is one @@ -1523,7 +2080,7 @@ public: * Remove following patterns: * mov x0, XX * mov x1, XX - * bl MCC_IncDecRef_NaiveRCFast + * bl MCC_IncDecRef_NaiveRCFast */ class RemoveIncRefPattern : public CGPeepPattern { public: @@ -1571,6 +2128,42 @@ private: std::vector optInsn; }; +// pattern1 : +// ----------------------------------------------------- +// lsr(304) R327 R324 8 +// strb(462) R327 [R164, 10] rev16 R327 R324 +// strb(462) R324 [R164, 11] => strh R327 [R164 10] +// pattern2 : +// ldrb(362) R369 R163 7 +// ldrb(362) R371 R163 6 ldrh R369 R163 6 +// add(157) R374 R369 R371 LSL => rev16 R374 R369 + +class LdrStrRevPattern : public CGPeepPattern { +public: + LdrStrRevPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn) : CGPeepPattern(cgFunc, currBB, currInsn) {} + ~LdrStrRevPattern() override + { + lsrInsn = nullptr; + adjacentInsn = nullptr; + curMemOpnd = nullptr; + adjacentMemOpnd = nullptr; + } + void Run(BB &bb, Insn &insn) override; + bool CheckCondition(Insn &insn) override; + std::string GetPatternName() override + { + return "LdrStrRevPattern"; + } + +private: + bool IsAdjacentMem(const MemOperand &memOperandLow, const MemOperand &memOperandHigh) const; + Insn *lsrInsn = nullptr; + Insn *adjacentInsn = nullptr; + MemOperand *curMemOpnd = nullptr; + MemOperand *adjacentMemOpnd = nullptr; + bool isStrInsn = false; +}; + /* * add x0, x1, #:lo12:Ljava_2Futil_2FLocale_241_3B_7C_24SwitchMap_24java_24util_24Locale_24Category * ldr x2, [x0] @@ -1732,6 +2325,110 @@ private: FuncNameOperand *target = nullptr; }; +/* + * Replace following patterns: + * + * add w1, w0, w1 + * cmp w1, #0 ====> adds w1, w0, w1 + * EQ + * + * add x1, x0, x1 + * cmp x1, #0 ====> adds x1, x0, x1 + *.......EQ + * + * .... + */ +class AddCmpZeroPattern : public CGPeepPattern { +public: + AddCmpZeroPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn) : CGPeepPattern(cgFunc, currBB, currInsn) {} + ~AddCmpZeroPattern() override + { + prevInsn = nullptr; + } + void Run(BB &bb, Insn &insn) override; + bool CheckCondition(Insn &insn) override; + std::string GetPatternName() override + { + return "AddCmpZeroPattern"; + } + +private: + bool CheckAddCmpZeroCheckAdd(const Insn &insn) const; + bool CheckAddCmpZeroContinue(const Insn &insn, const RegOperand &opnd) const; + bool CheckAddCmpZeroCheckCond(const Insn &insn) const; + Insn *prevInsn = nullptr; +}; + +/* + * Replace following pattern: + * sxtw x1, w0 + * lsl x2, x1, #3 ====> sbfiz x2, x0, #3, #32 + * + * uxtw x1, w0 + * lsl x2, x1, #3 ====> ubfiz x2, x0, #3, #32 + */ +class ComplexExtendWordLslAArch64 : public PeepPattern { +public: + explicit ComplexExtendWordLslAArch64(CGFunc &cgFunc) : PeepPattern(cgFunc) {} + ~ComplexExtendWordLslAArch64() override = default; + void Run(BB &bb, Insn &insn) override; + +private: + bool IsExtendWordLslPattern(const Insn &insn) const; +}; + +class ComplexExtendWordLslPattern : public CGPeepPattern { +public: + ComplexExtendWordLslPattern(CGFunc &cgFunc, BB &currBB, Insn &currInsn) : CGPeepPattern(cgFunc, currBB, currInsn) {} + ~ComplexExtendWordLslPattern() override + { + useInsn = nullptr; + } + void Run(BB &bb, Insn &insn) override; + bool CheckCondition(Insn &insn) override; + std::string GetPatternName() override + { + return "ComplexExtendWordLslPattern"; + } + +private: + Insn *useInsn = nullptr; +}; + +/* + * Replace following patterns: + * + * add w1, w0, w1 + * cmp w1, #0 ====> adds w1, w0, w1 + * EQ + * + * add x1, x0, x1 + * cmp x1, #0 ====> adds x1, x0, x1 + * EQ + * + * .... + */ +class AddCmpZeroPatternSSA : public CGPeepPattern { +public: + AddCmpZeroPatternSSA(CGFunc &cgFunc, BB &currBB, Insn &currInsn, CGSSAInfo &info) + : CGPeepPattern(cgFunc, currBB, currInsn, info) + { + } + ~AddCmpZeroPatternSSA() override + { + prevAddInsn = nullptr; + } + void Run(BB &bb, Insn &insn) override; + bool CheckCondition(Insn &insn) override; + std::string GetPatternName() override + { + return "AddCmpZeroPatternSSA"; + } + +private: + Insn *prevAddInsn = nullptr; +}; + /* * Optimize the following patterns: * and w0, w6, #1 ====> tbz w6, 0, .label @@ -1814,6 +2511,7 @@ public: explicit AndCmpCsetEorCbzOpt(CGFunc &cgFunc) : PeepPattern(cgFunc), cgFunc(&cgFunc) {} ~AndCmpCsetEorCbzOpt() override = default; void Run(BB &bb, Insn &insn) override; + private: CGFunc *cgFunc; }; @@ -1830,6 +2528,7 @@ public: explicit AddLdrOpt(CGFunc &cgFunc) : PeepPattern(cgFunc), cgFunc(&cgFunc) {} ~AddLdrOpt() override = default; void Run(BB &bb, Insn &insn) override; + private: CGFunc *cgFunc; }; @@ -1846,6 +2545,7 @@ public: explicit CsetEorOpt(CGFunc &cgFunc) : PeepPattern(cgFunc), cgFunc(&cgFunc) {} ~CsetEorOpt() override = default; void Run(BB &bb, Insn &insn) override; + private: CGFunc *cgFunc; }; @@ -1862,26 +2562,9 @@ public: explicit MoveCmpOpt(CGFunc &cgFunc) : PeepPattern(cgFunc), cgFunc(&cgFunc) {} ~MoveCmpOpt() override = default; void Run(BB &bb, Insn &insn) override; -private: - CGFunc *cgFunc; -}; - -/* - * Replace following pattern: - * sxtw x1, w0 - * lsl x2, x1, #3 ====> sbfiz x2, x0, #3, #32 - * - * uxtw x1, w0 - * lsl x2, x1, #3 ====> ubfiz x2, x0, #3, #32 - */ -class ComplexExtendWordLslAArch64 : public PeepPattern { -public: - explicit ComplexExtendWordLslAArch64(CGFunc &cgFunc) : PeepPattern(cgFunc) {} - ~ComplexExtendWordLslAArch64() override = default; - void Run(BB &bb, Insn &insn) override; private: - bool IsExtendWordLslPattern(const Insn &insn) const; + CGFunc *cgFunc; }; class AArch64PeepHole : public PeepPatternMatch { diff --git a/ecmascript/compiler/codegen/maple/maple_be/include/cg/cfgo.h b/ecmascript/compiler/codegen/maple/maple_be/include/cg/cfgo.h index 9dbf1a7db5c3ead21b44a97ffb96f2f05f65069d..f81a5b00e0ce0722b3e79d754b7d1802062ff6ea 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/include/cg/cfgo.h +++ b/ecmascript/compiler/codegen/maple/maple_be/include/cg/cfgo.h @@ -19,6 +19,14 @@ #include "optimize_common.h" namespace maplebe { + +enum CfgoPhase : maple::uint8 { + kCfgoDefault, + kCfgoPreRegAlloc, + kCfgoPostRegAlloc, + kPostCfgo, +}; + class ChainingPattern : public OptimizationPattern { public: explicit ChainingPattern(CGFunc &func) : OptimizationPattern(func) @@ -26,8 +34,8 @@ public: patternName = "BB Chaining"; dotColor = kCfgoChaining; } + ~ChainingPattern() override = default; - virtual ~ChainingPattern() = default; bool Optimize(BB &curBB) override; protected: @@ -48,12 +56,14 @@ public: dotColor = kCfgoSj; } - virtual ~SequentialJumpPattern() = default; + ~SequentialJumpPattern() override = default; bool Optimize(BB &curBB) override; protected: - void SkipSucBB(BB &curBB, BB &sucBB); - void UpdateSwitchSucc(BB &curBB, BB &sucBB); + void SkipSucBB(BB &curBB, BB &sucBB) const; + void UpdateSwitchSucc(BB &curBB, BB &sucBB) const; + // If the sucBB has one invalid predBB, the sucBB can not be skipped + bool HasInvalidPred(BB &sucBB) const; }; class FlipBRPattern : public OptimizationPattern { @@ -64,9 +74,19 @@ public: dotColor = kCfgoFlipCond; } - virtual ~FlipBRPattern() = default; + ~FlipBRPattern() override = default; bool Optimize(BB &curBB) override; + CfgoPhase GetPhase() const + { + return phase; + } + void SetPhase(CfgoPhase val) + { + phase = val; + } + CfgoPhase phase = kCfgoDefault; + protected: void RelocateThrowBB(BB &curBB); @@ -75,24 +95,27 @@ private: virtual MOperator FlipConditionOp(MOperator flippedOp) = 0; }; -/* This class represents the scenario that the BB is unreachable. */ +// This class represents the scenario that the BB is unreachable. class UnreachBBPattern : public OptimizationPattern { public: explicit UnreachBBPattern(CGFunc &func) : OptimizationPattern(func) { patternName = "Unreachable BB"; dotColor = kCfgoUnreach; - func.GetTheCFG()->FindAndMarkUnreachable(*cgFunc); } - virtual ~UnreachBBPattern() = default; + ~UnreachBBPattern() override = default; + + void InitPattern() override + { + cgFunc->GetTheCFG()->FindAndMarkUnreachable(*cgFunc); + } + bool Optimize(BB &curBB) override; }; -/* - * This class represents the scenario that a common jump BB can be duplicated - * to one of its another predecessor. - */ +// This class represents the scenario that a common jump BB can be duplicated +// to one of its another predecessor. class DuplicateBBPattern : public OptimizationPattern { public: explicit DuplicateBBPattern(CGFunc &func) : OptimizationPattern(func) @@ -101,16 +124,14 @@ public: dotColor = kCfgoDup; } - virtual ~DuplicateBBPattern() = default; + ~DuplicateBBPattern() override = default; bool Optimize(BB &curBB) override; private: static constexpr int kThreshold = 10; }; -/* - * This class represents the scenario that a BB contains nothing. - */ +// This class represents the scenario that a BB contains nothing. class EmptyBBPattern : public OptimizationPattern { public: explicit EmptyBBPattern(CGFunc &func) : OptimizationPattern(func) @@ -119,8 +140,57 @@ public: dotColor = kCfgoEmpty; } - virtual ~EmptyBBPattern() = default; + ~EmptyBBPattern() override = default; + bool Optimize(BB &curBB) override; +}; + +/* This class represents that two pred of a BB can cross-jump + * BB1: insn1 BB1: insn1 + * insn2 b BB4 (branch newBB) + * insn3 BB2: insn4 (fallthru, no need branch) + * b BB3 BB4: insn2 + * BB2: insn4 ==> insn3 + * insn2 b BB3 + * insn3 + * b BB3 + * BB1 & BB2 is BB3's pred, and has same insns(insn2, insn3). + * In BB2, we can split it into two BBs and set the newBB to be the fallthru of BB2. + * In BB1, same insns need to be deleted, and a jump insn pointing to the new BB is + * generated at the end of the BB. + */ +class CrossJumpBBPattern : public OptimizationPattern { +public: + explicit CrossJumpBBPattern(CGFunc &func) : OptimizationPattern(func) + { + patternName = "Cross Jump BB"; + dotColor = kCfgoEmpty; + } + ~CrossJumpBBPattern() override = default; + bool Optimize(BB &curBB) override; + +private: + enum MergeDirection { + kDirectionNone, + kDirectionForward, + kDirectionBackward, + kDirectionBoth, + }; + bool OptimizeOnce(const BB &curBB); + bool TryCrossJumpBB(BB &bb1, BB &bb2, MergeDirection dir = kDirectionBoth); + bool CheckBBSuccMatch(BB &bb1, BB &bb2); + uint32 CheckBBInsnsMatch(BB &bb1, BB &bb2, Insn *&f1, Insn *&f2); + void MergeMemInfo(BB &bb1, Insn &newpos1, BB &redirectBB); + void GetMergeDirection(BB &bb1, BB &bb2, const Insn &f1, const Insn &f2, MergeDirection &dir); + + BB &SplitBB(BB &srcBB, Insn &lastInsn); + + virtual uint32 GetJumpTargetIdx(const Insn &insn) const = 0; + virtual MOperator FlipConditionOp(MOperator flippedOp) const = 0; + virtual MOperator GetUnCondBranchMOP() const = 0; + + static constexpr uint32 kMaxCrossJumpPreds = 50; // limit for compiler time + static constexpr uint32 kMinCrossJumpPreds = 2; // Noting to do if there is not at least two incoming edges }; class CFGOptimizer : public Optimizer { @@ -130,13 +200,26 @@ public: name = "CFGO"; } - virtual ~CFGOptimizer() = default; + ~CFGOptimizer() override = default; + CfgoPhase GetPhase() const + { + return phase; + } + void SetPhase(CfgoPhase val) + { + phase = val; + } + +protected: + CfgoPhase phase = kCfgoDefault; }; +MAPLE_FUNC_PHASE_DECLARE_BEGIN(CgPreCfgo, maplebe::CGFunc) +MAPLE_FUNC_PHASE_DECLARE_END MAPLE_FUNC_PHASE_DECLARE_BEGIN(CgCfgo, maplebe::CGFunc) MAPLE_FUNC_PHASE_DECLARE_END MAPLE_FUNC_PHASE_DECLARE_BEGIN(CgPostCfgo, maplebe::CGFunc) MAPLE_FUNC_PHASE_DECLARE_END -} /* namespace maplebe */ +} // namespace maplebe -#endif /* MAPLEBE_INCLUDE_CG_CFGO_H */ +#endif // MAPLEBE_INCLUDE_CG_CFGO_H \ No newline at end of file diff --git a/ecmascript/compiler/codegen/maple/maple_be/include/cg/cfi.h b/ecmascript/compiler/codegen/maple/maple_be/include/cg/cfi.h index c0e2ed7cc2872993b38ae1c666a1d7c683f7218d..8ba347dae044d171a9f9d005af1a9b5eb3c0a0f0 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/include/cg/cfi.h +++ b/ecmascript/compiler/codegen/maple/maple_be/include/cg/cfi.h @@ -61,6 +61,8 @@ class CfiInsn : public maplebe::Insn { public: CfiInsn(MemPool &memPool, maplebe::MOperator op) : Insn(memPool, op) {} + CfiInsn(const CfiInsn &other) : maplebe::Insn(other) {} + CfiInsn(MemPool &memPool, maplebe::MOperator op, maplebe::Operand &opnd0) : Insn(memPool, op, opnd0) {} CfiInsn(MemPool &memPool, maplebe::MOperator op, maplebe::Operand &opnd0, maplebe::Operand &opnd1) @@ -76,6 +78,13 @@ public: ~CfiInsn() = default; + CfiInsn *CloneTree(MapleAllocator &allocator) const override + { + auto *insn = allocator.GetMemPool()->New(*this); + insn->DeepClone(*this, allocator); + return insn; + } + bool IsMachineInstruction() const override { return false; @@ -129,6 +138,12 @@ public: { return regNO; } + + RegOperand *CloneTree(MapleAllocator &allocator) const override + { + return allocator.GetMemPool()->New(*this); + } + Operand *Clone(MemPool &memPool) const override { Operand *opnd = memPool.Clone(*this); @@ -154,6 +169,12 @@ public: ~ImmOperand() = default; using OperandVisitable::OperandVisitable; + ImmOperand *CloneTree(MapleAllocator &allocator) const override + { + // const MIRSymbol is not changed in cg, so we can do shallow copy + return allocator.GetMemPool()->New(*this); + } + Operand *Clone(MemPool &memPool) const override { Operand *opnd = memPool.Clone(*this); @@ -184,6 +205,12 @@ public: ~SymbolOperand() = default; using OperandVisitable::OperandVisitable; + SymbolOperand *CloneTree(MapleAllocator &allocator) const override + { + // const MIRSymbol is not changed in cg, so we can do shallow copy + return allocator.GetMemPool()->New(*this); + } + Operand *Clone(MemPool &memPool) const override { Operand *opnd = memPool.Clone(*this); @@ -212,6 +239,11 @@ public: ~StrOperand() = default; using OperandVisitable::OperandVisitable; + StrOperand *CloneTree(MapleAllocator &allocator) const override + { + return allocator.GetMemPool()->New(*this); + } + Operand *Clone(MemPool &memPool) const override { Operand *opnd = memPool.Clone(*this); @@ -245,6 +277,11 @@ public: ~LabelOperand() = default; using OperandVisitable::OperandVisitable; + LabelOperand *CloneTree(MapleAllocator &allocator) const override + { + return allocator.GetMemPool()->New(*this); + } + Operand *Clone(MemPool &memPool) const override { Operand *opnd = memPool.Clone(*this); diff --git a/ecmascript/compiler/codegen/maple/maple_be/include/cg/cg_cfg.h b/ecmascript/compiler/codegen/maple/maple_be/include/cg/cg_cfg.h index a593c8709cd35d66609658da002a8c4dab4e3b8b..fde98ee04d7b5515388ea9e8692921c887620ae2 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/include/cg/cg_cfg.h +++ b/ecmascript/compiler/codegen/maple/maple_be/include/cg/cg_cfg.h @@ -60,7 +60,17 @@ public: virtual LabelIdx GetJumpLabel(const Insn &insn) const = 0; virtual bool IsCompareInsn(const Insn &insn) const = 0; virtual bool IsCompareAndBranchInsn(const Insn &insn) const = 0; + virtual bool IsTestAndSetCCInsn(const Insn &insn) const = 0; + virtual bool IsTestAndBranchInsn(const Insn &insn) const = 0; virtual bool IsAddOrSubInsn(const Insn &insn) const = 0; + virtual bool IsSimpleJumpInsn(const Insn &insn) const = 0; + + virtual void ReTargetSuccBB(BB &bb, LabelIdx newTarget) const = 0; + virtual void FlipIfBB(BB &bb, LabelIdx ftLabel) const = 0; + virtual BB *CreateGotoBBAfterCondBB(BB &bb, BB &fallthru, bool isTargetFallthru) const = 0; + + // Change ftBB to gotoBB, Append new jumpInsn in curBB. + virtual void ModifyFathruBBToGotoBB(BB &bb, LabelIdx labelIdx) const = 0; private: CGFunc *cgFunc; @@ -99,14 +109,21 @@ public: /* Skip the successor of bb, directly jump to bb's successor'ssuccessor */ void RetargetJump(BB &srcBB, BB &targetBB); + /* + * Update the preds of CommonExitBB after changing cfg, + * We'd better do it once after cfgo opt + */ + void UpdateCommonExitBBInfo(); + /* Loop up if the given label is in the exception tables in LSDA */ - static bool InLSDA(LabelIdx label, const EHFunc &ehFunc); + static bool InLSDA(LabelIdx label, const EHFunc *ehFunc); static bool InSwitchTable(LabelIdx label, const CGFunc &func); RegOperand *CreateVregFromReg(const RegOperand &pReg); Insn *CloneInsn(Insn &originalInsn); static BB *GetTargetSuc(BB &curBB, bool branchOnly = false, bool isGotoIf = false); bool IsCompareAndBranchInsn(const Insn &insn) const; + bool IsTestAndBranchInsn(const Insn &insn) const; bool IsAddOrSubInsn(const Insn &insn) const; Insn *FindLastCondBrInsn(BB &bb) const; diff --git a/ecmascript/compiler/codegen/maple/maple_be/include/cg/cg_ssa.h b/ecmascript/compiler/codegen/maple/maple_be/include/cg/cg_ssa.h index e907623c7c710ff2e4e6a91753459d64a9075c65..cbbb227724e12220cd0057c0256647968f2eac5c 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/include/cg/cg_ssa.h +++ b/ecmascript/compiler/codegen/maple/maple_be/include/cg/cg_ssa.h @@ -221,6 +221,7 @@ public: { return safePropInsns; } + Insn *GetDefInsn(const RegOperand &useReg); void DumpFuncCGIRinSSAForm() const; virtual void DumpInsnInSSAForm(const Insn &insn) const = 0; static uint32 SSARegNObase; diff --git a/ecmascript/compiler/codegen/maple/maple_be/include/cg/cgbb.h b/ecmascript/compiler/codegen/maple/maple_be/include/cg/cgbb.h index 4f6461fc63757d55c27c6b7ee42e9ba81f9ccee1..1558f9fd1b64fc1d818bcae6116efccec9bcc36b 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/include/cg/cgbb.h +++ b/ecmascript/compiler/codegen/maple/maple_be/include/cg/cgbb.h @@ -58,6 +58,8 @@ namespace maplebe { #define FOR_BB_INSNS_REV(INSN, BLOCK) \ for (Insn * (INSN) = LAST_INSN(BLOCK); (INSN) != nullptr; (INSN) = (INSN)->GetPrev()) +#define FOR_BB_INSNS_REV_CONST(INSN, BLOCK) \ + for (const Insn *(INSN) = LAST_INSN(BLOCK); (INSN) != nullptr; (INSN) = (INSN)->GetPrev()) /* For iterating over insns in basic block when we might remove the current insn. */ #define FOR_BB_INSNS_SAFE(INSN, BLOCK, NEXT) \ @@ -75,6 +77,8 @@ using BBID = uint32; class BB { public: + static constexpr int32 kUnknownProb = -1; + static constexpr uint32 kBBIfSuccsSize = 2; enum BBKind : uint8 { kBBFallthru, /* default */ kBBIf, /* conditional branch */ @@ -97,6 +101,7 @@ public: ehSuccs(mallocator.Adapter()), loopPreds(mallocator.Adapter()), loopSuccs(mallocator.Adapter()), + succsProb(mallocator.Adapter()), liveInRegNO(mallocator.Adapter()), liveOutRegNO(mallocator.Adapter()), callInsns(mallocator.Adapter()), @@ -158,6 +163,7 @@ public: if (next != nullptr) { next->prev = &bb; } + succsProb[&bb] = kUnknownProb; next = &bb; } @@ -169,6 +175,7 @@ public: this->prev->next = &bb; } this->prev = &bb; + succsProb[&bb] = kUnknownProb; } Insn *InsertInsnBefore(Insn &existing, Insn &newInsn); @@ -259,6 +266,7 @@ public: for (auto i = succs.begin(); i != succs.end(); ++i) { if (*i == &bb) { succs.erase(i); + succsProb.erase(&bb); return; } } @@ -380,6 +388,17 @@ public: } return nullptr; } + + const Insn *GetFirstMachineInsn() const + { + FOR_BB_INSNS_CONST(insn, this) { + if (insn->IsMachineInstruction()) { + return insn; + } + } + return nullptr; + } + Insn *GetLastMachineInsn() { FOR_BB_INSNS_REV(insn, this) { @@ -389,6 +408,17 @@ public: } return nullptr; } + + const Insn *GetLastMachineInsn() const + { + FOR_BB_INSNS_REV_CONST(insn, this) { + if (insn->IsMachineInstruction() && !insn->IsPseudo()) { + return insn; + } + } + return nullptr; + } + Insn *GetLastInsn() { return lastInsn; @@ -409,18 +439,27 @@ public: { preds.insert(it, &bb); } - void InsertSucc(const MapleList::iterator &it, BB &bb) + void InsertSucc(const MapleList::iterator &it, BB &bb, int32 prob = kUnknownProb) { succs.insert(it, &bb); + succsProb[&bb] = prob; } const MapleList &GetPreds() const { return preds; } + std::size_t GetPredsSize() const + { + return preds.size(); + } const MapleList &GetSuccs() const { return succs; } + std::size_t GetSuccsSize() const + { + return succs.size(); + } const MapleList &GetEhPreds() const { return ehPreds; @@ -477,9 +516,10 @@ public: { preds.push_back(&bb); } - void PushBackSuccs(BB &bb) + void PushBackSuccs(BB &bb, int32 prob = kUnknownProb) { succs.push_back(&bb); + succsProb[&bb] = prob; } void PushBackEhPreds(BB &bb) { @@ -501,15 +541,16 @@ public: { preds.push_front(&bb); } - void PushFrontSuccs(BB &bb) + void PushFrontSuccs(BB &bb, int32 prob = kUnknownProb) { succs.push_front(&bb); + succsProb[&bb] = prob; } void ErasePreds(MapleList::iterator it) { preds.erase(it); } - void EraseSuccs(MapleList::iterator it) + void EraseSuccs(MapleList::const_iterator it) { succs.erase(it); } @@ -520,6 +561,20 @@ public: void RemoveSuccs(BB &bb) { succs.remove(&bb); + succsProb.erase(&bb); + } + void ReplaceSucc(const MapleList::const_iterator it, BB &newBB) + { + int prob = succsProb[*it]; + EraseSuccs(it); + PushBackSuccs(newBB, prob); + } + + void ReplaceSucc(BB &oldBB, BB &newBB) + { + int prob = succsProb[&oldBB]; + RemoveSuccs(oldBB); + PushBackSuccs(newBB, prob); } void RemoveEhPreds(BB &bb) { @@ -536,6 +591,7 @@ public: void ClearSuccs() { succs.clear(); + succsProb.clear(); } void ClearEhPreds() { @@ -919,6 +975,47 @@ public: return alignNopNum; } + // Check if a given BB mergee can be merged into this BB in the sense of control flow + // The condition is looser than CanMerge, since we only need this for debug info. + bool MayFoldInCfg(const BB &mergee) const + { + bool onePred = (mergee.GetPreds().size() == 1) && (mergee.GetPreds().front() == this); + bool oneSucc = (GetSuccs().size() == 1) && (GetSuccs().front() == &mergee); + return oneSucc && onePred; + } + + // This means two bb are close to each other in cfg. Only for debug info. + // May add more conditions in the future. + bool IsCloseTo(const BB &tar) const + { + return MayFoldInCfg(tar); + } + + int32 GetEdgeProb(const BB &bb) const + { + return succsProb.find(&bb)->second; + } + + bool HasMachineInsn() + { + FOR_BB_INSNS(insn, this) { + if (insn->IsMachineInstruction()) { + return true; + } + } + return false; + } + + bool IsAdrpLabel() const + { + return isAdrpLabel; + } + + void SetIsAdrpLabel() + { + isAdrpLabel = true; + } + private: static const std::string bbNames[kBBLast]; uint32 id; @@ -940,6 +1037,7 @@ private: MapleList ehSuccs; MapleList loopPreds; MapleList loopSuccs; + MapleMap succsProb; /* this is for live in out analysis */ MapleSet liveInRegNO; @@ -1006,7 +1104,9 @@ private: bool needAlign = false; uint32 alignPower = 0; uint32 alignNopNum = 0; -}; /* class BB */ + + bool isAdrpLabel = false; // Indicate whether the address of this BB is referenced by adrp_label insn +}; /* class BB */ struct BBIdCmp { bool operator()(const BB *lhs, const BB *rhs) const diff --git a/ecmascript/compiler/codegen/maple/maple_be/include/cg/cgfunc.h b/ecmascript/compiler/codegen/maple/maple_be/include/cg/cgfunc.h index abe2f00a69162233fdd21818b5d55edb49b70662..6043b767f04b6c7b045f511812e26934044b80e3 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/include/cg/cgfunc.h +++ b/ecmascript/compiler/codegen/maple/maple_be/include/cg/cgfunc.h @@ -957,6 +957,21 @@ public: return exitBBVec.at(index); } + MapleVector::iterator EraseNoReturnCallBB(MapleVector::iterator it) + { + return noReturnCallBBVec.erase(it); + } + + void PushBackNoReturnCallBBsVec(BB &bb) + { + noReturnCallBBVec.emplace_back(&bb); + } + + MapleVector &GetNoRetCallBBVec() + { + return noReturnCallBBVec; + } + void SetLab2BBMap(int32 index, BB &bb) { lab2BBMap[index] = &bb; @@ -1619,6 +1634,7 @@ private: BB *commonExitBB = nullptr; /* this post-dominate all BBs */ Insn *volReleaseInsn = nullptr; /* use to record the release insn for volatile strore */ MapleVector exitBBVec; + MapleVector noReturnCallBBVec; MapleSet extendSet; /* use to mark regs which spilled 32 bits but loaded 64 bits. */ MapleUnorderedMap lab2BBMap; BECommon &beCommon; diff --git a/ecmascript/compiler/codegen/maple/maple_be/include/cg/dbg.h b/ecmascript/compiler/codegen/maple/maple_be/include/cg/dbg.h index 61b3c1f55e250a30880f90573b46447369a8ea1e..56b834f8c6b5d7dd0e248889a0042c823a98f637 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/include/cg/dbg.h +++ b/ecmascript/compiler/codegen/maple/maple_be/include/cg/dbg.h @@ -40,6 +40,8 @@ class DbgInsn : public maplebe::Insn { public: DbgInsn(MemPool &memPool, maplebe::MOperator op) : Insn(memPool, op) {} + DbgInsn(const DbgInsn &other) : maplebe::Insn(other) {} + DbgInsn(MemPool &memPool, maplebe::MOperator op, maplebe::Operand &opnd0) : Insn(memPool, op, opnd0) {} DbgInsn(MemPool &memPool, maplebe::MOperator op, maplebe::Operand &opnd0, maplebe::Operand &opnd1) @@ -55,6 +57,13 @@ public: ~DbgInsn() = default; + DbgInsn *CloneTree(MapleAllocator &allocator) const override + { + auto *insn = allocator.GetMemPool()->New(*this); + insn->DeepClone(*this, allocator); + return insn; + } + bool IsMachineInstruction() const override { return false; @@ -76,6 +85,11 @@ public: return true; } + bool IsDbgLine() const override + { + return mOp == OP_DBG_loc; + } + bool IsRegDefined(maplebe::regno_t regNO) const override { CHECK_FATAL(false, "dbg insn do not def regs"); @@ -106,6 +120,12 @@ public: ~ImmOperand() = default; using OperandVisitable::OperandVisitable; + ImmOperand *CloneTree(MapleAllocator &allocator) const override + { + // const MIRSymbol is not changed in cg, so we can do shallow copy + return allocator.GetMemPool()->New(*this); + } + Operand *Clone(MemPool &memPool) const override { Operand *opnd = memPool.Clone(*this); diff --git a/ecmascript/compiler/codegen/maple/maple_be/include/cg/insn.h b/ecmascript/compiler/codegen/maple/maple_be/include/cg/insn.h index 15a55536caca4cf875eead1b84b468abb2f39667..041255eaa4a9797a3890b6bbee6a023140850f26 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/include/cg/insn.h +++ b/ecmascript/compiler/codegen/maple/maple_be/include/cg/insn.h @@ -20,7 +20,9 @@ #include #include #include +/* Maple CG headers */ #include "operand.h" +#include "isa.h" #include "stackmap.h" #include "mpl_logging.h" #include "sparse_datainfo.h" @@ -29,6 +31,8 @@ #include "types_def.h" /* for uint32 */ #include "common_utils.h" +/* Maple Util headers */ +#include "mem_reference_table.h" /* for alias */ namespace maplebe { /* forward declaration */ class BB; @@ -84,6 +88,22 @@ public: } virtual ~Insn() = default; + void DeepClone(const Insn &insn, MapleAllocator &allocator) + { + opnds.clear(); + for (auto opnd : insn.opnds) { + opnds.emplace_back(opnd->CloneTree(allocator)); + } + } + + // Custom deep copy + virtual Insn *CloneTree(MapleAllocator &allocator) const + { + auto *insn = allocator.GetMemPool()->New(*this); + insn->DeepClone(*this, allocator); + return insn; + } + MOperator GetMachineOpcode() const { return mOp; @@ -135,6 +155,14 @@ public: opnds[index] = &opnd; } + // Get size info from machine description + uint32 GetOperandSize(uint32 index) const + { + CHECK_FATAL(index < opnds.size(), "index out of range!"); + const OpndDesc *opndMD = md->GetOpndDes(index); + return opndMD->GetSize(); + } + void SetRetSize(uint32 size) { DEBUG_ASSERT(IsCall(), "Insn should be a call."); @@ -147,6 +175,11 @@ public: return retSize; } + void SplitSelf(bool isAfterRegAlloc, InsnBuilder *insnBuilder, OperandBuilder *opndBuilder) + { + md->Split(this, isAfterRegAlloc, insnBuilder, opndBuilder); + } + virtual bool IsMachineInstruction() const; virtual bool IsIntrinsic() const @@ -165,9 +198,12 @@ public: Operand *GetMemOpnd() const; + uint32 GetMemOpndIdx() const; + void SetMemOpnd(MemOperand *memOpnd); bool IsCall() const; + bool IsSpecialCall() const; bool IsTailCall() const; bool IsAsmInsn() const; bool IsClinit() const; @@ -231,6 +267,11 @@ public: return false; } + virtual bool IsDbgLine() const + { + return false; + } + bool IsDMBInsn() const; bool IsVectorOp() const; @@ -276,6 +317,11 @@ public: isFrameDef = b; } + void SetStackDef(bool flag) + { + isStackDef = flag; + } + bool IsAsmDefCondCode() const { return asmDefCondCode; @@ -581,6 +627,41 @@ public: return registerBinding; } + void SetReferenceOsts(MemDefUse *memDefUse) + { + referenceOsts = memDefUse; + } + + const MemDefUse *GetReferenceOsts() const + { + return referenceOsts; + } + + void MergeReferenceOsts(Insn &rhs) + { + if (referenceOsts == nullptr) { + SetReferenceOsts(rhs.referenceOsts); + } else if (rhs.referenceOsts != nullptr) { + referenceOsts->MergeOthers(*rhs.referenceOsts); + } + } + + bool Equals(const Insn &rhs) const + { + if (&rhs == this) { + return true; + } + if (mOp != rhs.mOp || opnds.size() != rhs.opnds.size()) { + return false; + } + for (int i = 0; i < opnds.size(); ++i) { + if (!opnds[i]->Equals(*rhs.opnds[i])) { + return false; + } + } + return true; + } + void AddDeoptBundleInfo(int32 deoptVreg, Operand &opnd) { if (stackMap == nullptr) { @@ -694,6 +775,7 @@ private: bool isSpill = false; /* used as hint for optimization */ bool isReload = false; /* used as hint for optimization */ bool isFrameDef = false; + bool isStackDef = false; // def sp in prolog bool asmDefCondCode = false; bool asmModMem = false; bool needSplit = false; @@ -703,6 +785,7 @@ private: /* for multiple architecture */ const InsnDesc *md = nullptr; + MemDefUse *referenceOsts = nullptr; SparseDataInfo *stackMapDef = nullptr; SparseDataInfo *stackMapUse = nullptr; SparseDataInfo *stackMapLiveIn = nullptr; diff --git a/ecmascript/compiler/codegen/maple/maple_be/include/cg/isa.h b/ecmascript/compiler/codegen/maple/maple_be/include/cg/isa.h index 3f32ba08fb4633f3ffdaa7d32ad3b6e1b905ac19..b1e3d1f187fc472515344ecc23a601e2132a085b 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/include/cg/isa.h +++ b/ecmascript/compiler/codegen/maple/maple_be/include/cg/isa.h @@ -21,6 +21,14 @@ #include "operand.h" namespace maplebe { +#define SPLIT_INSN(INSN, FUNC) \ + (INSN)->SplitSelf((FUNC)->IsAfterRegAlloc(), (FUNC)->GetInsnBuilder(), (FUNC)->GetOpndBuilder()) + +// circular dependency exists, no other choice +class Insn; +class InsnBuilder; +class OperandBuilder; + enum MopProperty : maple::uint8 { kInsnIsAbstract, kInsnIsMove, @@ -28,8 +36,10 @@ enum MopProperty : maple::uint8 { kInsnIsLoadPair, kInsnIsStore, kInsnIsStorePair, + kInsnIsLoadAddress, kInsnIsAtomic, kInsnIsCall, + kInsnIsSpecialCall, kInsnIsTailCall, kInsnIsConversion, kInsnIsCondDef, @@ -60,8 +70,10 @@ using regno_t = uint32_t; #define ISLOADPAIR (1ULL << kInsnIsLoadPair) #define ISSTORE (1ULL << kInsnIsStore) #define ISSTOREPAIR (1ULL << kInsnIsStorePair) +#define ISLOADADDR (1ULL << kInsnIsLoadAddress) #define ISATOMIC (1ULL << kInsnIsAtomic) #define ISCALL (1ULL << kInsnIsCall) +#define ISSPCALL (1ULL << kInsnIsSpecialCall) #define ISTAILCALL (1ULL << kInsnIsTailCall) #define ISCONVERSION (1ULL << kInsnIsConversion) #define ISCONDDEF (1ULL << kInsnIsCondDef) @@ -172,6 +184,8 @@ enum EncodeType : uint8 { }; struct InsnDesc { + using SplitFunc = std::function; + InsnDesc(MOperator op, std::vector opndmd, uint64 props, uint64 ltype, const std::string &inName, const std::string &inFormat, uint32 anum) : opc(op), @@ -229,6 +243,8 @@ struct InsnDesc { const std::string format; uint32 atomicNum; /* indicate how many asm instructions it will emit. */ std::function validFunc = nullptr; /* If insn has immOperand, this function needs to be implemented. */ + // If insn needs to be split, this function needs to be implemented. + SplitFunc splitFunc = nullptr; EncodeType encodeType = kUnknownEncodeType; uint32 mopEncode = 0x00000000; @@ -238,6 +254,11 @@ struct InsnDesc { { return (properties & ISCALL) != 0; } + // call insn does not obey standard call procedure! + bool IsSpecialCall() const + { + return (properties & ISSPCALL) != 0; + } bool IsTailCall() const { return properties & ISTAILCALL; @@ -294,6 +315,10 @@ struct InsnDesc { { return (properties & (ISUNCONDBRANCH)) != 0; } + bool IsLoadAddress() const + { + return (properties & (ISLOADADDR)) != 0; + } bool IsAtomic() const { return (properties & ISATOMIC) != 0; @@ -359,6 +384,14 @@ struct InsnDesc { return opc; } + void Split(Insn *insn, bool isAfterRegAlloc, InsnBuilder *insnBuilder, OperandBuilder *opndBuilder) const + { + if (!splitFunc) { + return; + } + splitFunc(insn, isAfterRegAlloc, insnBuilder, opndBuilder); + } + const OpndDesc *GetOpndDes(size_t index) const { return opndMD[index]; diff --git a/ecmascript/compiler/codegen/maple/maple_be/include/cg/loop.h b/ecmascript/compiler/codegen/maple/maple_be/include/cg/loop.h index e2cae274adf4af2125ec848b1c8283068ff4da78..7d28a4f2c3fbe0270204e5d340603ec406881ad4 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/include/cg/loop.h +++ b/ecmascript/compiler/codegen/maple/maple_be/include/cg/loop.h @@ -197,6 +197,7 @@ public: void CheckOverlappingInnerLoops(const MapleVector &iLoops, const MapleVector &loopMem) const; void CheckLoops() const; void PrintLoops(const CGFuncLoops &funcLoop) const; + bool IsBBLoopMember(const BB *bb); const BB *GetHeader() const { diff --git a/ecmascript/compiler/codegen/maple/maple_be/include/cg/operand.h b/ecmascript/compiler/codegen/maple/maple_be/include/cg/operand.h index 90dfdb8e503e7ad7f5c8ff0ea2e78ddad03a6b80..a8495916c2461e798922aa40f930ddff5bedbb68 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/include/cg/operand.h +++ b/ecmascript/compiler/codegen/maple/maple_be/include/cg/operand.h @@ -190,6 +190,8 @@ public: return false; } + // Custom deep copy + virtual Operand *CloneTree(MapleAllocator &allocator) const = 0; virtual Operand *Clone(MemPool &memPool) const = 0; /* @@ -246,6 +248,11 @@ public: ~RegOperand() override = default; using OperandVisitable::OperandVisitable; + RegOperand *CloneTree(MapleAllocator &allocator) const override + { + return allocator.GetMemPool()->New(*this); + } + Operand *Clone(MemPool &memPool) const override { return memPool.Clone(*this); @@ -543,6 +550,12 @@ public: ~ImmOperand() override = default; using OperandVisitable::OperandVisitable; + ImmOperand *CloneTree(MapleAllocator &allocator) const override + { + // const MIRSymbol is not changed in cg, so we can do shallow copy + return allocator.GetMemPool()->New(*this); + } + Operand *Clone(MemPool &memPool) const override { return memPool.Clone(*this); @@ -830,6 +843,12 @@ public: symbol = nullptr; } + OfstOperand *CloneTree(MapleAllocator &allocator) const override + { + // const MIRSymbol is not changed in cg, so we can do shallow copy + return allocator.GetMemPool()->New(*this); + } + Operand *Clone(MemPool &memPool) const override { return memPool.Clone(*this); @@ -1100,7 +1119,8 @@ public: idxOpt(memOpnd.idxOpt), noExtend(memOpnd.noExtend), isStackMem(memOpnd.isStackMem), - isStackArgMem(memOpnd.isStackArgMem) + isStackArgMem(memOpnd.isStackArgMem), + isVolatile(memOpnd.isVolatile) { } @@ -1109,6 +1129,21 @@ public: ~MemOperand() override = default; using OperandVisitable::OperandVisitable; + MemOperand *CloneTree(MapleAllocator &allocator) const override + { + auto *memOpnd = allocator.GetMemPool()->New(*this); + if (baseOpnd != nullptr) { + memOpnd->SetBaseRegister(*baseOpnd->CloneTree(allocator)); + } + if (indexOpnd != nullptr) { + memOpnd->SetIndexRegister(*indexOpnd->CloneTree(allocator)); + } + if (offsetOpnd != nullptr) { + memOpnd->SetOffsetOperand(*offsetOpnd->CloneTree(allocator)); + } + return memOpnd; + } + MemOperand *Clone(MemPool &memPool) const override { return memPool.Clone(*this); @@ -1318,7 +1353,8 @@ public: { return (GetSize() == opnd.GetSize()) && (addrMode == opnd.addrMode) && (extend == opnd.extend) && (GetBaseRegister() == opnd.GetBaseRegister()) && (GetIndexRegister() == opnd.GetIndexRegister()) && - (GetSymbol() == opnd.GetSymbol()) && (GetOffsetOperand() == opnd.GetOffsetOperand()); + (GetSymbol() == opnd.GetSymbol()) && (GetOffsetOperand() == opnd.GetOffsetOperand()) && + (IsVolatile() == opnd.IsVolatile()); } VaryType GetMemVaryType() const @@ -1398,6 +1434,11 @@ public: extend = val; } + void SetVolatile(bool flag) + { + isVolatile = flag; + } + bool IsIntactIndexed() const { return idxOpt == kIntact; @@ -1413,6 +1454,11 @@ public: return idxOpt == kPreIndex; } + bool IsVolatile() const + { + return isVolatile; + } + std::string GetExtendAsString() const { if (GetIndexRegister()->GetSize() == k64BitSize) { @@ -1440,6 +1486,7 @@ private: bool noExtend = false; bool isStackMem = false; bool isStackArgMem = false; + bool isVolatile = false; // based on mem info from ME }; class LabelOperand : public OperandVisitable { @@ -1452,6 +1499,11 @@ public: ~LabelOperand() override = default; using OperandVisitable::OperandVisitable; + LabelOperand *CloneTree(MapleAllocator &allocator) const override + { + return allocator.GetMemPool()->New(*this); + } + Operand *Clone(MemPool &memPool) const override { return memPool.Clone(*this); @@ -1534,6 +1586,15 @@ public: using OperandVisitable::OperandVisitable; + ListOperand *CloneTree(MapleAllocator &allocator) const override + { + auto *listOpnd = allocator.GetMemPool()->New(allocator); + for (auto regOpnd : opndList) { + listOpnd->PushOpnd(*regOpnd->CloneTree(allocator)); + } + return listOpnd; + } + Operand *Clone(MemPool &memPool) const override { return memPool.Clone(*this); @@ -1549,6 +1610,11 @@ public: return opndList; } + const MapleList &GetOperands() const + { + return opndList; + } + void Dump() const override { for (auto it = opndList.begin(); it != opndList.end();) { @@ -1592,6 +1658,12 @@ public: ~StImmOperand() override = default; using OperandVisitable::OperandVisitable; + StImmOperand *CloneTree(MapleAllocator &allocator) const override + { + // const MIRSymbol is not changed in cg, so we can do shallow copy + return allocator.GetMemPool()->New(*this); + } + Operand *Clone(MemPool &memPool) const override { return memPool.Clone(*this); @@ -1670,6 +1742,11 @@ public: ~ExtendShiftOperand() override = default; using OperandVisitable::OperandVisitable; + ExtendShiftOperand *CloneTree(MapleAllocator &allocator) const override + { + return allocator.GetMemPool()->New(*this); + } + Operand *Clone(MemPool &memPool) const override { return memPool.Clone(*this); @@ -1715,6 +1792,11 @@ public: ~BitShiftOperand() override = default; using OperandVisitable::OperandVisitable; + BitShiftOperand *CloneTree(MapleAllocator &allocator) const override + { + return allocator.GetMemPool()->New(*this); + } + Operand *Clone(MemPool &memPool) const override { return memPool.Clone(*this); @@ -1779,6 +1861,11 @@ public: return comment; } + CommentOperand *CloneTree(MapleAllocator &allocator) const override + { + return allocator.GetMemPool()->New(*this); + } + Operand *Clone(MemPool &memPool) const override { return memPool.Clone(*this); @@ -1824,6 +1911,15 @@ public: } } + ListConstraintOperand *CloneTree(MapleAllocator &allocator) const override + { + auto *constraintOpnd = allocator.GetMemPool()->New(allocator); + for (auto stringOpnd : stringList) { + constraintOpnd->stringList.emplace_back(stringOpnd->CloneTree(allocator)); + } + return constraintOpnd; + } + Operand *Clone(MemPool &memPool) const override { return memPool.Clone(*this); @@ -1853,6 +1949,15 @@ public: ~PhiOperand() override = default; using OperandVisitable::OperandVisitable; + PhiOperand *CloneTree(MapleAllocator &allocator) const override + { + auto *phiOpnd = allocator.GetMemPool()->New(allocator); + for (auto phiPair : phiList) { + phiOpnd->InsertOpnd(phiPair.first, *phiPair.second->CloneTree(allocator)); + } + return phiOpnd; + } + Operand *Clone(MemPool &memPool) const override { return memPool.Clone(*this); @@ -1938,6 +2043,12 @@ public: symbol = &fsym; } + FuncNameOperand *CloneTree(MapleAllocator &allocator) const override + { + // const MIRSymbol is not changed in cg, so we can do shallow copy + return allocator.GetMemPool()->New(*this); + } + Operand *Clone(MemPool &memPool) const override { return memPool.New(*this); @@ -1969,11 +2080,7 @@ private: namespace operand { /* bit 0-7 for common */ -enum CommOpndDescProp : maple::uint64 { - kIsDef = 1ULL, - kIsUse = (1ULL << 1), - kIsVector = (1ULL << 2) -}; +enum CommOpndDescProp : maple::uint64 { kIsDef = 1ULL, kIsUse = (1ULL << 1), kIsVector = (1ULL << 2) }; /* bit 8-15 for reg */ enum RegOpndDescProp : maple::uint64 { @@ -1984,8 +2091,7 @@ enum RegOpndDescProp : maple::uint64 { }; /* bit 16-23 for imm */ -enum ImmOpndDescProp : maple::uint64 { -}; +enum ImmOpndDescProp : maple::uint64 {}; /* bit 24-31 for mem */ enum MemOpndDescProp : maple::uint64 { @@ -2082,6 +2188,11 @@ public: ~CondOperand() override = default; using OperandVisitable::OperandVisitable; + CondOperand *CloneTree(MapleAllocator &allocator) const override + { + return allocator.GetMemPool()->New(*this); + } + Operand *Clone(MemPool &memPool) const override { return memPool.New(cc); diff --git a/ecmascript/compiler/codegen/maple/maple_be/include/cg/optimize_common.h b/ecmascript/compiler/codegen/maple/maple_be/include/cg/optimize_common.h index 1cce12846b12840daac1a171fa8aa75cf0c85846..10634d8315114f213eb1742ab01db3d8d9e89b4d 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/include/cg/optimize_common.h +++ b/ecmascript/compiler/codegen/maple/maple_be/include/cg/optimize_common.h @@ -25,8 +25,10 @@ inline const std::string kCfgoAlways = "green"; inline const std::string kCfgoUnreach = "yellow"; inline const std::string kCfgoDup = "orange"; inline const std::string kCfgoEmpty = "purple"; +inline const std::string kCfgoCrossJump = "brown"; inline const std::string kIcoIte = "blue"; /* if conversion optimization, if-then-else */ inline const std::string kIcoIt = "grey"; /* if conversion optimization, if-then-else */ +inline const std::string kDup = "maize"; /* dup optimization */ class OptimizationPattern { public: @@ -48,12 +50,17 @@ public: bool IsLabelInLSDAOrSwitchTable(LabelIdx label) const { +#ifndef ONLY_C EHFunc *ehFunc = cgFunc->GetEHFunc(); - return (ehFunc != nullptr && cgFunc->GetTheCFG()->InLSDA(label, *ehFunc)) || + return (ehFunc != nullptr && cgFunc->GetTheCFG()->InLSDA(label, ehFunc)) || cgFunc->GetTheCFG()->InSwitchTable(label, *cgFunc); +#else + return CGCFG::InSwitchTable(label, *cgFunc); +#endif } - void Search2Op(bool noOptimize); + bool Search2Op(bool noOptimize); + virtual void InitPattern() {} virtual bool Optimize(BB &curBB) = 0; protected: diff --git a/ecmascript/compiler/codegen/maple/maple_be/include/cg/peep.h b/ecmascript/compiler/codegen/maple/maple_be/include/cg/peep.h index fa4258ca6d97ab8b82e0aeafac502a995790a07e..849da97fc431f7d05d9f165e431637318d2ae3c0 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/include/cg/peep.h +++ b/ecmascript/compiler/codegen/maple/maple_be/include/cg/peep.h @@ -40,7 +40,7 @@ public: } OptimizePattern optPattern(*cgFunc, *currBB, *currInsn, *ssaInfo); optPattern.Run(*currBB, *currInsn); - optSuccess = optPattern.GetPatternRes(); + optSuccess = optPattern.GetPatternRes() || optSuccess; if (optSuccess && optPattern.GetCurrInsn() != nullptr) { currInsn = optPattern.GetCurrInsn(); } @@ -53,6 +53,14 @@ public: } OptimizePattern optPattern(*cgFunc, *currBB, *currInsn); optPattern.Run(*currBB, *currInsn); + optSuccess = optPattern.GetPatternRes() || optSuccess; + if (optSuccess && optPattern.GetCurrInsn() != nullptr) { + currInsn = optPattern.GetCurrInsn(); + } + } + void SetOptSuccess(bool optRes) + { + optSuccess = optRes; } bool OptSuccess() const { @@ -64,6 +72,13 @@ private: BB *currBB; Insn *currInsn; CGSSAInfo *ssaInfo; + /* + * The flag indicates whether the optimization pattern is successful, + * this prevents the next optimization pattern that processs the same mop from failing to get the validInsn, + * which was changed by previous pattern. + * + * Set the flag to true when the pattern optimize successfully. + */ bool optSuccess = false; }; @@ -75,7 +90,10 @@ public: CGPeepHole(CGFunc &f, MemPool *memPool, CGSSAInfo *cgssaInfo) : cgFunc(&f), peepMemPool(memPool), ssaInfo(cgssaInfo) { } - ~CGPeepHole() = default; + virtual ~CGPeepHole() + { + ssaInfo = nullptr; + } virtual void Run() = 0; virtual bool DoSSAOptimize(BB &bb, Insn &insn) = 0; @@ -99,7 +117,7 @@ public: bool CheckOpndLiveinSuccs(const RegOperand ®Opnd, const BB &bb) const; bool CheckRegLiveinReturnBB(const RegOperand ®Opnd, const BB &bb) const; ReturnType IsOpndLiveinBB(const RegOperand ®Opnd, const BB &bb) const; - int logValueAtBase2(int64 val) const; + int LogValueAtBase2(int64 val) const; bool IsMemOperandOptPattern(const Insn &insn, Insn &nextInsn); protected: @@ -125,10 +143,10 @@ public: virtual std::string GetPatternName() = 0; Insn *GetDefInsn(const RegOperand &useReg); void DumpAfterPattern(std::vector &prevInsns, const Insn *replacedInsn, const Insn *newInsn); - InsnSet GetAllUseInsn(const RegOperand &defReg); + InsnSet GetAllUseInsn(const RegOperand &defReg) const; int64 GetLogValueAtBase2(int64 val) const; /* The CC reg is unique and cannot cross-version props. */ - bool IsCCRegCrossVersion(Insn &startInsn, Insn &endInsn, const RegOperand &ccReg); + bool IsCCRegCrossVersion(Insn &startInsn, Insn &endInsn, const RegOperand &ccReg) const; /* optimization support function */ bool IfOperandIsLiveAfterInsn(const RegOperand ®Opnd, Insn &insn); bool FindRegLiveOut(const RegOperand ®Opnd, const BB &bb); @@ -155,6 +173,8 @@ protected: BB *currBB; Insn *currInsn; CGSSAInfo *ssaInfo; + // !!! If the pattern is optimized, set the $optSuccess to true and check before the subsequent patterns + // of the same mop, otherwise, the subsequent patterns of the same mop will get the old wrong instruction. bool optSuccess = false; }; @@ -172,7 +192,7 @@ public: private: CGFunc *cgFunc; - CG *cg; + CG *cg = nullptr; }; /* class PeepHoleOptimizer */ class PeepPatternMatch { diff --git a/ecmascript/compiler/codegen/maple/maple_be/include/cg/x86_64/x64_optimize_common.h b/ecmascript/compiler/codegen/maple/maple_be/include/cg/x86_64/x64_optimize_common.h index 0924e38120554dd00b3d239537f328cb99b99a6f..042270da352f4c92a696fcc1e084e29fb9c3a9a0 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/include/cg/x86_64/x64_optimize_common.h +++ b/ecmascript/compiler/codegen/maple/maple_be/include/cg/x86_64/x64_optimize_common.h @@ -36,8 +36,15 @@ public: LabelIdx GetJumpLabel(const Insn &insn) const override; bool IsCompareInsn(const Insn &insn) const override; bool IsCompareAndBranchInsn(const Insn &insn) const override; + bool IsTestAndSetCCInsn(const Insn &insn) const override; + bool IsTestAndBranchInsn(const Insn &insn) const override; bool IsAddOrSubInsn(const Insn &insn) const override; + bool IsSimpleJumpInsn(const Insn &insn) const override; RegOperand *CreateVregFromReg(const RegOperand &pReg) override; + void ReTargetSuccBB(BB &bb, LabelIdx newTarget) const override; + void FlipIfBB(BB &bb, LabelIdx ftLabel) const override; + BB *CreateGotoBBAfterCondBB(BB &bb, BB &fallthru, bool isTargetFallthru) const override; + void ModifyFathruBBToGotoBB(BB &bb, LabelIdx labelIdx) const override; }; } /* namespace maplebe */ diff --git a/ecmascript/compiler/codegen/maple/maple_be/src/cg/aarch64/aarch64_cfgo.cpp b/ecmascript/compiler/codegen/maple/maple_be/src/cg/aarch64/aarch64_cfgo.cpp index 2d8605db25d424db467f6cc4516f9684738c7d73..d15f774d4b55e7f6292994c0a01d9bbdb43f7bfb 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/src/cg/aarch64/aarch64_cfgo.cpp +++ b/ecmascript/compiler/codegen/maple/maple_be/src/cg/aarch64/aarch64_cfgo.cpp @@ -14,14 +14,20 @@ */ #include "aarch64_cfgo.h" -#include "aarch64_isa.h" namespace maplebe { /* Initialize cfg optimization patterns */ void AArch64CFGOptimizer::InitOptimizePatterns() { diffPassPatterns.emplace_back(memPool->New(*cgFunc)); - diffPassPatterns.emplace_back(memPool->New(*cgFunc)); + if (cgFunc->GetMirModule().IsCModule()) { + diffPassPatterns.emplace_back(memPool->New(*cgFunc)); + } + auto *brOpt = memPool->New(*cgFunc); + if (GetPhase() == kCfgoPostRegAlloc) { + brOpt->SetPhase(kCfgoPostRegAlloc); + } + diffPassPatterns.emplace_back(brOpt); diffPassPatterns.emplace_back(memPool->New(*cgFunc)); diffPassPatterns.emplace_back(memPool->New(*cgFunc)); diffPassPatterns.emplace_back(memPool->New(*cgFunc)); @@ -31,8 +37,19 @@ uint32 AArch64FlipBRPattern::GetJumpTargetIdx(const Insn &insn) { return AArch64isa::GetJumpTargetIdx(insn); } + MOperator AArch64FlipBRPattern::FlipConditionOp(MOperator flippedOp) { return AArch64isa::FlipConditionOp(flippedOp); } + +uint32 AArch64CrossJumpBBPattern::GetJumpTargetIdx(const Insn &insn) const +{ + return AArch64isa::GetJumpTargetIdx(insn); +} + +MOperator AArch64CrossJumpBBPattern::FlipConditionOp(MOperator flippedOp) const +{ + return AArch64isa::FlipConditionOp(flippedOp); +} } // namespace maplebe diff --git a/ecmascript/compiler/codegen/maple/maple_be/src/cg/aarch64/aarch64_dependence.cpp b/ecmascript/compiler/codegen/maple/maple_be/src/cg/aarch64/aarch64_dependence.cpp index 5c7d8536f283cf464e12d07effc952517beaf242..fa3734e871d781f4643cd5a7cfd2c2c8083cfddf 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/src/cg/aarch64/aarch64_dependence.cpp +++ b/ecmascript/compiler/codegen/maple/maple_be/src/cg/aarch64/aarch64_dependence.cpp @@ -256,7 +256,7 @@ void AArch64DepAnalysis::CombineMemoryAccessPair(DepNode &firstNode, DepNode &se DEBUG_ASSERT(firstNode.GetInsn(), "the insn of first Node should not be nullptr"); DEBUG_ASSERT(secondNode.GetInsn(), "the insn of second Node should not be nullptr"); MOperator thisMop = firstNode.GetInsn()->GetMachineOpcode(); - MOperator mopPair = GetMopPair(thisMop); + MOperator mopPair = GetMopPair(thisMop, false); DEBUG_ASSERT(mopPair != 0, "mopPair should not be zero"); Operand *opnd0 = nullptr; Operand *opnd1 = nullptr; @@ -1146,7 +1146,8 @@ void AArch64DepAnalysis::Run(BB &bb, MapleVector &nodes) uint32 nodeSum = 1; MapleVector comments(alloc.Adapter()); const Insn *locInsn = bb.GetFirstLoc(); - FOR_BB_INSNS(insn, (&bb)) { + FOR_BB_INSNS(insn, (&bb)) + { if (!insn->IsMachineInstruction()) { if (insn->IsImmaterialInsn()) { if (!insn->IsComment()) { diff --git a/ecmascript/compiler/codegen/maple/maple_be/src/cg/aarch64/aarch64_ico.cpp b/ecmascript/compiler/codegen/maple/maple_be/src/cg/aarch64/aarch64_ico.cpp index 8f4300024f504459e0037fbf3274b0cacd5b03d8..75ccaf726816ea748f2c31675f943795d10bccf4 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/src/cg/aarch64/aarch64_ico.cpp +++ b/ecmascript/compiler/codegen/maple/maple_be/src/cg/aarch64/aarch64_ico.cpp @@ -310,7 +310,8 @@ bool AArch64ICOIfThenElsePattern::BuildCondMovInsn(BB &cmpBB, const BB &bb, bool elseBBIsProcessed, std::vector &generateInsn) { Insn *branchInsn = cgFunc->GetTheCFG()->FindLastCondBrInsn(cmpBB); - FOR_BB_INSNS_CONST(insn, (&bb)) { + FOR_BB_INSNS_CONST(insn, (&bb)) + { if (!insn->IsMachineInstruction() || insn->IsBranch()) { continue; } @@ -468,7 +469,8 @@ bool AArch64ICOIfThenElsePattern::CheckCondMoveBB(BB *bb, std::mapIsMachineInstruction() || insn->IsBranch()) { continue; } @@ -649,7 +651,7 @@ bool AArch64ICOIfThenElsePattern::DoOpt(BB &cmpBB, BB *ifBB, BB *elseBB, BB &joi } if (cmpBB.GetKind() != BB::kBBIf && cmpBB.GetNext() == &joinBB && - !maplebe::CGCFG::InLSDA(joinBB.GetLabIdx(), *cgFunc->GetEHFunc()) && + !maplebe::CGCFG::InLSDA(joinBB.GetLabIdx(), cgFunc->GetEHFunc()) && cgFunc->GetTheCFG()->CanMerge(cmpBB, joinBB)) { maplebe::CGCFG::MergeBB(cmpBB, joinBB, *cgFunc); keepPosition = true; @@ -690,13 +692,12 @@ bool AArch64ICOIfThenElsePattern::Optimize(BB &curBB) return false; } DEBUG_ASSERT(elseBB != nullptr, "elseBB should not be nullptr"); - if (CGCFG::InLSDA(elseBB->GetLabIdx(), *cgFunc->GetEHFunc()) || - CGCFG::InSwitchTable(elseBB->GetLabIdx(), *cgFunc)) { + if (CGCFG::InLSDA(elseBB->GetLabIdx(), cgFunc->GetEHFunc()) || CGCFG::InSwitchTable(elseBB->GetLabIdx(), *cgFunc)) { return false; } if (ifBB != nullptr && - (CGCFG::InLSDA(ifBB->GetLabIdx(), *cgFunc->GetEHFunc()) || CGCFG::InSwitchTable(ifBB->GetLabIdx(), *cgFunc))) { + (CGCFG::InLSDA(ifBB->GetLabIdx(), cgFunc->GetEHFunc()) || CGCFG::InSwitchTable(ifBB->GetLabIdx(), *cgFunc))) { return false; } return DoOpt(curBB, ifBB, elseBB, *joinBB); @@ -776,7 +777,8 @@ bool AArch64ICOSameCondPattern::DoOpt(BB *firstIfBB, BB &secondIfBB) } /* secondifBB only has branchInsn and cmpInsn */ - FOR_BB_INSNS_REV(insn, &secondIfBB) { + FOR_BB_INSNS_REV(insn, &secondIfBB) + { if (!insn->IsMachineInstruction()) { continue; } @@ -847,7 +849,8 @@ bool AArch64ICOMorePredsPattern::Optimize(BB &curBB) /* this BBGoto only has mov Insn and Branch */ bool AArch64ICOMorePredsPattern::CheckGotoBB(BB &gotoBB, std::vector &movInsn) const { - FOR_BB_INSNS(insn, &gotoBB) { + FOR_BB_INSNS(insn, &gotoBB) + { if (!insn->IsMachineInstruction()) { continue; } diff --git a/ecmascript/compiler/codegen/maple/maple_be/src/cg/aarch64/aarch64_isa.cpp b/ecmascript/compiler/codegen/maple/maple_be/src/cg/aarch64/aarch64_isa.cpp index b24732334a5310b091d37156d68c5663a1d9dd48..f439ec5a3abb2a428853faaccfb626574207bbf2 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/src/cg/aarch64/aarch64_isa.cpp +++ b/ecmascript/compiler/codegen/maple/maple_be/src/cg/aarch64/aarch64_isa.cpp @@ -21,7 +21,7 @@ namespace maplebe { * Get the ldp/stp corresponding to ldr/str * mop : a ldr or str machine operator */ -MOperator GetMopPair(MOperator mop) +MOperator GetMopPair(MOperator mop, bool isIncludeStrbStrh) { switch (mop) { case MOP_xldr: @@ -44,6 +44,10 @@ MOperator GetMopPair(MOperator mop) return MOP_sstp; case MOP_qstr: return MOP_qstp; + case MOP_wstrb: + return isIncludeStrbStrh ? MOP_wstrh : MOP_undef; + case MOP_wstrh: + return isIncludeStrbStrh ? MOP_wstr : MOP_undef; default: DEBUG_ASSERT(false, "should not run here"); return MOP_undef; @@ -106,7 +110,7 @@ uint32 GetJumpTargetIdx(const Insn &insn) return kInsnFirstOpnd; } case MOP_xbr: { - DEBUG_ASSERT(insn.GetOperandSize() == 2, "ERR"); // must have 2 + DEBUG_ASSERT(insn.GetOperandSize() == 2, "ERR"); // must have 2 return kInsnSecondOpnd; } /* conditional jump */ @@ -141,5 +145,21 @@ uint32 GetJumpTargetIdx(const Insn &insn) } return kInsnFirstOpnd; } + +// This api is only used for cgir verify, implemented by calling the memopndofst interface. +int64 GetMemOpndOffsetValue(Operand *o) +{ + auto *memOpnd = static_cast(o); + CHECK_FATAL(memOpnd != nullptr, "memOpnd should not be nullptr"); + // kBOR memOpnd has no offsetvalue, so return 0 for verify. + // todo: AArch64AddressingMode is different from BiShengC + if (memOpnd->GetAddrMode() == MemOperand::kAddrModeBOrX) { + return 0; + } + // Offset value of kBOI & kLo12Li can be got. + OfstOperand *ofStOpnd = memOpnd->GetOffsetImmediate(); + int64 offsetValue = ofStOpnd ? ofStOpnd->GetOffsetValue() : 0LL; + return offsetValue; +} } /* namespace AArch64isa */ } /* namespace maplebe */ diff --git a/ecmascript/compiler/codegen/maple/maple_be/src/cg/aarch64/aarch64_optimize_common.cpp b/ecmascript/compiler/codegen/maple/maple_be/src/cg/aarch64/aarch64_optimize_common.cpp index dd6f819d936945f2bd003d34b2d1d9e13b31967e..59ef620efd37e90e759fbf542c545b08e3cbb50c 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/src/cg/aarch64/aarch64_optimize_common.cpp +++ b/ecmascript/compiler/codegen/maple/maple_be/src/cg/aarch64/aarch64_optimize_common.cpp @@ -14,7 +14,7 @@ */ #include "aarch64_optimize_common.h" -#include "aarch64_isa.h" +#include "aarch64_cg.h" #include "aarch64_cgfunc.h" #include "cgbb.h" @@ -45,7 +45,8 @@ void AArch64InsnVisitor::ModifyJumpTarget(Operand &targetOperand, BB &bb) } // fallthru below to patch the branch insn } - bb.GetLastInsn()->SetOperand(AArch64isa::GetJumpTargetIdx(*bb.GetLastInsn()), targetOperand); + CHECK_NULL_FATAL(bb.GetLastMachineInsn()); + bb.GetLastMachineInsn()->SetOperand(AArch64isa::GetJumpTargetIdx(*bb.GetLastMachineInsn()), targetOperand); } void AArch64InsnVisitor::ModifyJumpTarget(maple::LabelIdx targetLabel, BB &bb) @@ -55,27 +56,28 @@ void AArch64InsnVisitor::ModifyJumpTarget(maple::LabelIdx targetLabel, BB &bb) void AArch64InsnVisitor::ModifyJumpTarget(BB &newTarget, BB &bb) { - ModifyJumpTarget(newTarget.GetLastInsn()->GetOperand(AArch64isa::GetJumpTargetIdx(*newTarget.GetLastInsn())), bb); + if (newTarget.GetLastMachineInsn() != nullptr) { + ModifyJumpTarget(newTarget.GetLastInsn()->GetOperand(AArch64isa::GetJumpTargetIdx(*newTarget.GetLastInsn())), + bb); + } } Insn *AArch64InsnVisitor::CloneInsn(Insn &originalInsn) { + // Use custom deep copy + MapleAllocator *allocator = GetCGFunc()->GetFuncScopeAllocator(); MemPool *memPool = const_cast(CG::GetCurCGFunc()->GetMemoryPool()); - if (originalInsn.IsTargetInsn()) { - if (!originalInsn.IsVectorOp()) { - return memPool->Clone(originalInsn); - } else { + if (originalInsn.IsTargetInsn() || originalInsn.IsComment()) { + if (originalInsn.IsVectorOp()) { auto *insn = memPool->Clone(*static_cast(&originalInsn)); insn->SetRegSpecList(static_cast(originalInsn).GetRegSpecList()); return insn; } + return originalInsn.CloneTree(*allocator); } else if (originalInsn.IsCfiInsn()) { - return memPool->Clone(*static_cast(&originalInsn)); + return static_cast(originalInsn).CloneTree(*allocator); } else if (originalInsn.IsDbgInsn()) { - return memPool->Clone(*static_cast(&originalInsn)); - } - if (originalInsn.IsComment()) { - return memPool->Clone(originalInsn); + return static_cast(originalInsn).CloneTree(*allocator); } CHECK_FATAL(false, "Cannot clone"); return nullptr; @@ -102,8 +104,10 @@ bool AArch64InsnVisitor::IsCompareInsn(const Insn &insn) const switch (insn.GetMachineOpcode()) { case MOP_wcmpri: case MOP_wcmprr: + case MOP_wcmprrs: case MOP_xcmpri: case MOP_xcmprr: + case MOP_xcmprrs: case MOP_hcmperi: case MOP_hcmperr: case MOP_scmperi: @@ -120,6 +124,12 @@ bool AArch64InsnVisitor::IsCompareInsn(const Insn &insn) const case MOP_wcmnrr: case MOP_xcmnri: case MOP_xcmnrr: + case MOP_wwcmprre: + case MOP_xwcmprre: + case MOP_wccmpriic: + case MOP_wccmprric: + case MOP_xccmpriic: + case MOP_xccmprric: return true; default: return false; @@ -139,6 +149,32 @@ bool AArch64InsnVisitor::IsCompareAndBranchInsn(const Insn &insn) const } } +bool AArch64InsnVisitor::IsTestAndSetCCInsn(const Insn &insn) const +{ + switch (insn.GetMachineOpcode()) { + case MOP_wtstri32: + case MOP_xtstri64: + case MOP_wtstrr: + case MOP_xtstrr: + return true; + default: + return false; + } +} + +bool AArch64InsnVisitor::IsTestAndBranchInsn(const Insn &insn) const +{ + switch (insn.GetMachineOpcode()) { + case MOP_xtbz: + case MOP_wtbz: + case MOP_xtbnz: + case MOP_wtbnz: + return true; + default: + return false; + } +} + bool AArch64InsnVisitor::IsAddOrSubInsn(const Insn &insn) const { switch (insn.GetMachineOpcode()) { @@ -156,9 +192,76 @@ bool AArch64InsnVisitor::IsAddOrSubInsn(const Insn &insn) const } } +bool AArch64InsnVisitor::IsSimpleJumpInsn(const Insn &insn) const +{ + return (insn.GetMachineOpcode() == MOP_xuncond); +} + RegOperand *AArch64InsnVisitor::CreateVregFromReg(const RegOperand &pReg) { return &static_cast(GetCGFunc()) ->CreateRegisterOperandOfType(pReg.GetRegisterType(), pReg.GetSize() / k8BitSize); } + +void AArch64InsnVisitor::ReTargetSuccBB(BB &bb, LabelIdx newTarget) const +{ + Insn *lastInsn = bb.GetLastMachineInsn(); + if (lastInsn && (lastInsn->IsBranch() || lastInsn->IsCondBranch() || lastInsn->IsUnCondBranch())) { + CHECK_FATAL(false, "check last insn of a ft BB"); + } + LabelOperand &targetOpnd = GetCGFunc()->GetOrCreateLabelOperand(newTarget); + Insn &newInsn = GetCGFunc()->GetInsnBuilder()->BuildInsn(MOP_xuncond, targetOpnd); + bb.AppendInsn(newInsn); +} + +void AArch64InsnVisitor::FlipIfBB(BB &bb, LabelIdx ftLabel) const +{ + Insn *lastInsn = bb.GetLastMachineInsn(); + CHECK_FATAL(lastInsn && lastInsn->IsCondBranch(), "must be ? of a if BB"); + uint32 targetIdx = AArch64isa::GetJumpTargetIdx(*lastInsn); + MOperator mOp = AArch64isa::FlipConditionOp(lastInsn->GetMachineOpcode()); + if (mOp == 0 || mOp > MOP_nop) { + CHECK_FATAL(false, "get flip op failed"); + } + lastInsn->SetMOP(AArch64CG::kMd[mOp]); + LabelOperand &targetOpnd = GetCGFunc()->GetOrCreateLabelOperand(ftLabel); + lastInsn->SetOperand(targetIdx, targetOpnd); +} + +BB *AArch64InsnVisitor::CreateGotoBBAfterCondBB(BB &bb, BB &fallthru, bool isTargetFallthru) const +{ + BB *newBB = GetCGFunc()->CreateNewBB(); + newBB->SetKind(BB::kBBGoto); + LabelIdx fallthruLabel = fallthru.GetLabIdx(); + if (fallthruLabel == MIRLabelTable::GetDummyLabel()) { + fallthruLabel = GetCGFunc()->CreateLabel(); + fallthru.SetLabIdx(fallthruLabel); + } + LabelOperand &targetOpnd = GetCGFunc()->GetOrCreateLabelOperand(fallthruLabel); + Insn &gotoInsn = GetCGFunc()->GetInsnBuilder()->BuildInsn(MOP_xuncond, targetOpnd); + newBB->AppendInsn(gotoInsn); + + // maintain pred and succ + if (!isTargetFallthru) { + fallthru.RemovePreds(bb); + } + fallthru.PushBackPreds(*newBB); + if (!isTargetFallthru) { + bb.RemoveSuccs(fallthru); + } + bb.PushBackSuccs(*newBB); + newBB->PushBackSuccs(fallthru); + newBB->PushBackPreds(bb); + return newBB; +} + +void AArch64InsnVisitor::ModifyFathruBBToGotoBB(BB &bb, LabelIdx labelIdx) const +{ + CHECK_FATAL(bb.GetKind() == BB::kBBFallthru, "invalid kind of bb"); + CGFunc *cgFunc = GetCGFunc(); + LabelOperand &labelOpnd = cgFunc->GetOrCreateLabelOperand(labelIdx); + Insn &jumpInsn = cgFunc->GetInsnBuilder()->BuildInsn(MOP_xuncond, labelOpnd); + bb.AppendInsn(jumpInsn); + bb.SetKind(BB::kBBGoto); +} } /* namespace maplebe */ diff --git a/ecmascript/compiler/codegen/maple/maple_be/src/cg/aarch64/aarch64_peep.cpp b/ecmascript/compiler/codegen/maple/maple_be/src/cg/aarch64/aarch64_peep.cpp index d9ef0151ab786bc0ecbc59911f082c0611ae70f2..0dfb9da295ba89e947e389c88e950981038767ce 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/src/cg/aarch64/aarch64_peep.cpp +++ b/ecmascript/compiler/codegen/maple/maple_be/src/cg/aarch64/aarch64_peep.cpp @@ -19,6 +19,9 @@ #include "common_utils.h" #include "cg_option.h" #include "aarch64_utils.h" +#include "cg_irbuilder.h" +#include "aarch64_cg.h" +#include "aarch64_mem_reference.h" namespace maplebe { #define JAVALANG (cgFunc->GetMirModule().IsJavaModule()) @@ -72,16 +75,53 @@ static bool IsZeroRegister(const Operand &opnd) return regOpnd->GetRegisterNumber() == RZR; } +MOperator GetMopUpdateAPSR(MOperator mop, bool &isAddShift) +{ + MOperator newMop = MOP_undef; + // todo: the mop commented is in BiShengC + switch (mop) { + case MOP_xaddrrr: { + isAddShift = false; + break; + } + case MOP_xaddrri12: { + isAddShift = false; + break; + } + case MOP_waddrrr: { + isAddShift = false; + break; + } + case MOP_waddrri12: { + isAddShift = false; + break; + } + case MOP_xaddrrrs: { + isAddShift = true; + break; + } + case MOP_waddrrrs: { + isAddShift = true; + break; + } + default: + break; + } + return newMop; +} + void AArch64CGPeepHole::Run() { bool optSuccess = false; - FOR_ALL_BB(bb, cgFunc) { - FOR_BB_INSNS_SAFE(insn, bb, nextInsn) { + FOR_ALL_BB(bb, cgFunc) + { + FOR_BB_INSNS_SAFE(insn, bb, nextInsn) + { if (!insn->IsMachineInstruction()) { continue; } if (ssaInfo != nullptr) { - optSuccess = DoSSAOptimize(*bb, *insn); + optSuccess |= DoSSAOptimize(*bb, *insn); } else { DoNormalOptimize(*bb, *insn); } @@ -96,6 +136,7 @@ bool AArch64CGPeepHole::DoSSAOptimize(BB &bb, Insn &insn) { MOperator thisMop = insn.GetMachineOpcode(); manager = peepMemPool->New(*cgFunc, bb, insn, *ssaInfo); + manager->SetOptSuccess(false); switch (thisMop) { case MOP_xandrrr: case MOP_wandrrr: { @@ -119,15 +160,22 @@ bool AArch64CGPeepHole::DoSSAOptimize(BB &bb, Insn &insn) case MOP_beq: case MOP_bne: { manager->Optimize(true); + manager->Optimize(true); break; } case MOP_wcsetrc: case MOP_xcsetrc: { + manager->Optimize(true); + manager->Optimize(true); manager->Optimize(true); break; } case MOP_waddrrr: - case MOP_xaddrrr: + case MOP_xaddrrr: { + manager->Optimize(true); + manager->Optimize(true); + break; + } case MOP_dadd: case MOP_sadd: case MOP_wsubrrr: @@ -143,12 +191,17 @@ bool AArch64CGPeepHole::DoSSAOptimize(BB &bb, Insn &insn) } case MOP_wandrri12: case MOP_xandrri13: { + manager->Optimize(true); manager->Optimize(true); + manager->Optimize(true); + manager->Optimize(true); break; } case MOP_wcselrrrc: case MOP_xcselrrrc: { + manager->Optimize(true); manager->Optimize(true); + manager->Optimize(true); break; } case MOP_wiorrrr: @@ -173,6 +226,7 @@ bool AArch64CGPeepHole::DoSSAOptimize(BB &bb, Insn &insn) case MOP_xlslrri6: { manager->Optimize(); manager->Optimize(true); + manager->Optimize(true); break; } case MOP_xsxtb32: @@ -190,7 +244,6 @@ bool AArch64CGPeepHole::DoSSAOptimize(BB &bb, Insn &insn) case MOP_xlsrrri6: case MOP_wasrrri5: case MOP_xasrrri6: - case MOP_wlslrri5: case MOP_waddrri12: case MOP_xaddrri12: case MOP_wsubrri12: @@ -198,11 +251,35 @@ bool AArch64CGPeepHole::DoSSAOptimize(BB &bb, Insn &insn) manager->Optimize(true); break; } + case MOP_wlslrri5: { + manager->Optimize(true); + manager->Optimize(true); + break; + } case MOP_wubfxrri5i5: case MOP_xubfxrri6i6: { + manager->Optimize(true); manager->Optimize(true); break; } + case MOP_xmulrrr: + case MOP_wmulrrr: { + manager->Optimize(!cgFunc->IsAfterRegAlloc()); + break; + } + case MOP_wcmpri: + case MOP_xcmpri: { + manager->Optimize(true); + break; + } + case MOP_wtbz: + case MOP_xtbz: + case MOP_wtbnz: + case MOP_xtbnz: { + manager->Optimize(true); + manager->Optimize(true); + break; + } default: break; } @@ -236,7 +313,7 @@ bool ContinuousCmpCsetPattern::CheckCondition(Insn &insn) } reverse = (condOpnd.GetCode() == CC_EQ); auto &ccReg = static_cast(insn.GetOperand(kInsnThirdOpnd)); - prevCmpInsn = GetDefInsn(ccReg); + prevCmpInsn = ssaInfo->GetDefInsn(ccReg); if (prevCmpInsn == nullptr) { return false; } @@ -253,7 +330,7 @@ bool ContinuousCmpCsetPattern::CheckCondition(Insn &insn) return false; } auto &cmpUseReg = static_cast(prevCmpInsn->GetOperand(kInsnSecondOpnd)); - prevCsetInsn1 = GetDefInsn(cmpUseReg); + prevCsetInsn1 = ssaInfo->GetDefInsn(cmpUseReg); if (prevCsetInsn1 == nullptr) { return false; } @@ -346,7 +423,7 @@ bool NegCmpToCmnPattern::CheckCondition(Insn &insn) return false; } auto &useReg = static_cast(insn.GetOperand(kInsnThirdOpnd)); - prevInsn = GetDefInsn(useReg); + prevInsn = ssaInfo->GetDefInsn(useReg); if (prevInsn == nullptr) { return false; } @@ -354,6 +431,21 @@ bool NegCmpToCmnPattern::CheckCondition(Insn &insn) if (prevMop != MOP_winegrr && prevMop != MOP_xinegrr && prevMop != MOP_winegrrs && prevMop != MOP_xinegrrs) { return false; } + // Determine whether implicit conversion is existed. + if ((prevMop == MOP_winegrr && curMop == MOP_xcmprr) || (prevMop == MOP_winegrrs && curMop == MOP_xcmprr) || + (prevMop == MOP_xinegrr && curMop == MOP_wcmprr) || (prevMop == MOP_winegrrs && curMop == MOP_xcmprr)) { + return false; + } + /* + * If the value of srcOpnd of neg is 0, we can not do this optimization. + * Because + * for cmp(subs): if NOT(0), the add calculation may overflow + * for cmn(adds): it has handled overflows before calling the addWithCarry function + * When the value of srcOpnd of neg is 0, (neg + cmp) and (cmn) set different (c) and (v) in condition flags. + * + * But we can not get the value of register, so we can only restrict the condition codes which use (c) and (v) + * flags. + */ auto &ccReg = static_cast(insn.GetOperand(kInsnFirstOpnd)); InsnSet useInsns = GetAllUseInsn(ccReg); for (auto *useInsn : useInsns) { @@ -361,20 +453,27 @@ bool NegCmpToCmnPattern::CheckCondition(Insn &insn) continue; } MOperator useMop = useInsn->GetMachineOpcode(); - if (useMop == MOP_bhi || useMop == MOP_bls) { + if (useInsn->IsCondBranch() && useMop != MOP_beq && useMop != MOP_bne && useMop != MOP_bmi && + useMop != MOP_bpl) { return false; } - bool findUnsignedCond = false; - for (size_t i = 0; i < useInsn->GetOperandSize(); ++i) { + bool hasUnsupportedCode = false; + for (uint32 i = 0; i < useInsn->GetOperandSize(); ++i) { if (useInsn->GetOperand(i).GetKind() == Operand::kOpdCond) { ConditionCode cond = static_cast(useInsn->GetOperand(i)).GetCode(); - if (cond == CC_HI || cond == CC_LS) { - findUnsignedCond = true; + /* in case of ignoring v flag + * adds xt, x0, x1 (0x8000000000000000) -> not set v + * ==> + * neg x1 x1 (0x8000000000000000) which is same for negative 0 + * subs xt, x0, x1 () -> set v + */ + if (cond != CC_EQ && cond != CC_NE && cond != CC_MI && cond != CC_PL) { + hasUnsupportedCode = true; break; } } } - if (findUnsignedCond) { + if (hasUnsupportedCode) { return false; } } @@ -396,7 +495,6 @@ void NegCmpToCmnPattern::Run(BB &bb, Insn &insn) MOperator newMop = (currMop == MOP_wcmprr) ? MOP_wcmnrr : MOP_xcmnrr; newInsn = &(cgFunc->GetInsnBuilder()->BuildInsn(newMop, ccReg, opnd1, opnd2)); } else { - /* prevMop == MOP_winegrrs || prevMop == MOP_xinegrrs */ MOperator newMop = (currMop == MOP_wcmprr) ? MOP_wcmnrrs : MOP_xcmnrrs; Operand &shiftOpnd = prevInsn->GetOperand(kInsnThirdOpnd); newInsn = &(cgFunc->GetInsnBuilder()->BuildInsn(newMop, ccReg, opnd1, opnd2, shiftOpnd)); @@ -415,14 +513,168 @@ void NegCmpToCmnPattern::Run(BB &bb, Insn &insn) } } +void LdrCmpPattern::Run(BB &bb, Insn &insn) +{ + if (!CheckCondition(insn)) { + return; + } + bb.RemoveInsn(*ldr2); + bb.RemoveInsn(*ldr1); + bb.RemoveInsn(insn); + bb.RemoveInsn(*bne1); + prevLdr1->SetMOP(AArch64CG::kMd[MOP_xldr]); + prevLdr2->SetMOP(AArch64CG::kMd[MOP_xldr]); + prevCmp->SetMOP(AArch64CG::kMd[MOP_xcmprr]); +} + +bool LdrCmpPattern::CheckCondition(Insn &insn) +{ + /* a pattern which breaks cfg + * it is more suitable for peephole after pgo using */ + if (currInsn != &insn) { + return false; + } + if (!SetInsns()) { + return false; + } + if (!CheckInsns()) { + return false; + } + auto ®0 = static_cast(currInsn->GetOperand(kInsnSecondOpnd)); + auto ®1 = static_cast(currInsn->GetOperand(kInsnThirdOpnd)); + return !(IfOperandIsLiveAfterInsn(reg0, insn) || IfOperandIsLiveAfterInsn(reg1, insn)); +} + +/* + * mopSeq: + * ldr,ldr,cmp,bne + */ +bool LdrCmpPattern::SetInsns() +{ + if (!IsLdr(currInsn->GetPreviousMachineInsn())) { + return false; + } + ldr2 = currInsn->GetPreviousMachineInsn(); + if (!IsLdr(ldr2->GetPreviousMachineInsn())) { + return false; + } + ldr1 = ldr2->GetPreviousMachineInsn(); + /* ldr1 must be firstInsn in currBB */ + if (currInsn->GetBB()->GetFirstMachineInsn() != ldr1) { + return false; + } + if (!IsBne(currInsn->GetNextMachineInsn())) { + return false; + } + bne1 = currInsn->GetNextMachineInsn(); + BB *prevBB = currInsn->GetBB()->GetPrev(); + /* single prev, single pred */ + const MapleList &predBBs = currInsn->GetBB()->GetPreds(); + if ((prevBB == nullptr) || (predBBs.size() != 1) || (prevBB != *predBBs.begin())) { + return false; + } + if (!IsBne(prevBB->GetLastMachineInsn())) { + return false; + } + bne2 = prevBB->GetLastMachineInsn(); + if (!IsCmp(bne2->GetPreviousMachineInsn())) { + return false; + } + prevCmp = bne2->GetPreviousMachineInsn(); + prevLdr2 = prevCmp->GetPreviousMachineInsn(); + if (prevCmp == nullptr || prevLdr2 == nullptr) { + return false; + } + if (!IsLdr(prevCmp->GetPreviousMachineInsn())) { + return false; + } + if (!IsLdr(prevLdr2->GetPreviousMachineInsn())) { + return false; + } + prevLdr1 = prevLdr2->GetPreviousMachineInsn(); + return true; +} + +bool LdrCmpPattern::CheckInsns() const +{ + auto &label1 = static_cast(bne1->GetOperand(kInsnSecondOpnd)); + auto &label2 = static_cast(bne2->GetOperand(kInsnSecondOpnd)); + if (label1.GetLabelIndex() != label2.GetLabelIndex()) { + return false; + } + auto ®0 = static_cast(currInsn->GetOperand(kInsnSecondOpnd)); + auto ®1 = static_cast(currInsn->GetOperand(kInsnThirdOpnd)); + regno_t regno0 = reg0.GetRegisterNumber(); + regno_t regno1 = reg1.GetRegisterNumber(); + if (regno0 == regno1) { + return false; + } + auto &mem1 = static_cast(ldr1->GetOperand(kInsnSecondOpnd)); + auto &preMem1 = static_cast(prevLdr1->GetOperand(kInsnSecondOpnd)); + auto &mem2 = static_cast(ldr2->GetOperand(kInsnSecondOpnd)); + auto &preMem2 = static_cast(prevLdr2->GetOperand(kInsnSecondOpnd)); + regno_t regnoBase0 = mem1.GetBaseRegister()->GetRegisterNumber(); + regno_t regnoBase1 = mem2.GetBaseRegister()->GetRegisterNumber(); + if (regnoBase0 == regnoBase1) { + return false; + } + if ((regno0 == regnoBase0) || (regno0 == regnoBase1) || (regno1 == regnoBase0) || (regno1 == regnoBase1)) { + return false; + } + if ((reg0 == static_cast(ldr2->GetOperand(kInsnFirstOpnd))) && + (reg0 == static_cast(prevLdr2->GetOperand(kInsnFirstOpnd))) && + (reg1 == static_cast(ldr1->GetOperand(kInsnFirstOpnd))) && + (reg1 == static_cast(prevLdr1->GetOperand(kInsnFirstOpnd)))) { + if (MemOffet4Bit(preMem2, mem2) && MemOffet4Bit(preMem1, mem1)) { + return true; + } + } + if ((reg0 == static_cast(ldr1->GetOperand(kInsnFirstOpnd))) && + (reg0 == static_cast(prevLdr1->GetOperand(kInsnFirstOpnd))) && + (reg1 == static_cast(ldr2->GetOperand(kInsnFirstOpnd))) && + (reg1 == static_cast(prevLdr2->GetOperand(kInsnFirstOpnd)))) { + if (MemOffet4Bit(preMem2, mem2) && MemOffet4Bit(preMem1, mem1)) { + return true; + } + } + return false; +} + +bool LdrCmpPattern::MemOffet4Bit(const MemOperand &m1, const MemOperand &m2) const +{ + if (m1.GetAddrMode() != m2.GetAddrMode()) { + return false; + } + if (m1.GetAddrMode() != MemOperand::kAddrModeBOi) { + return false; + } + if (m1.GetBaseRegister()->GetRegisterNumber() != m2.GetBaseRegister()->GetRegisterNumber()) { + return false; + } + int64 offset = m2.GetOffsetOperand()->GetValue() - m1.GetOffsetOperand()->GetValue(); + return offset == k4BitSizeInt; +} + bool CsetCbzToBeqPattern::CheckCondition(Insn &insn) { MOperator curMop = insn.GetMachineOpcode(); - if (curMop != MOP_wcbz && curMop != MOP_xcbz && curMop != MOP_wcbnz && curMop != MOP_xcbnz) { + bool isValidMop = false; + if (curMop == MOP_wtbnz || curMop == MOP_xtbnz || curMop == MOP_wtbz || curMop == MOP_xtbz) { + auto &immOpnd1 = static_cast(insn.GetOperand(kInsnSecondOpnd)); + if (immOpnd1.GetValue() == 0) { + labelOpnd = &static_cast(insn.GetOperand(kInsnThirdOpnd)); + isValidMop = true; + } + } + if (curMop == MOP_wcbz || curMop == MOP_xcbz || curMop == MOP_wcbnz || curMop == MOP_xcbnz) { + labelOpnd = &static_cast(insn.GetOperand(kInsnSecondOpnd)); + isValidMop = true; + } + if (!isValidMop) { return false; } auto &useReg = static_cast(insn.GetOperand(kInsnFirstOpnd)); - prevInsn = GetDefInsn(useReg); + prevInsn = ssaInfo->GetDefInsn(useReg); if (prevInsn == nullptr) { return false; } @@ -481,12 +733,11 @@ void CsetCbzToBeqPattern::Run(BB &bb, Insn &insn) return; } MOperator curMop = insn.GetMachineOpcode(); - bool reverse = (curMop == MOP_wcbz || curMop == MOP_xcbz); - auto &labelOpnd = static_cast(insn.GetOperand(kInsnSecondOpnd)); + bool reverse = (curMop == MOP_wcbz || curMop == MOP_xcbz || curMop == MOP_wtbz || curMop == MOP_xtbz); auto &condOpnd = static_cast(prevInsn->GetOperand(kInsnSecondOpnd)); MOperator newMop = SelectNewMop(condOpnd.GetCode(), reverse); DEBUG_ASSERT(newMop != MOP_undef, "unknown condition code"); - Insn &newInsn = cgFunc->GetInsnBuilder()->BuildInsn(newMop, prevInsn->GetOperand(kInsnThirdOpnd), labelOpnd); + Insn &newInsn = cgFunc->GetInsnBuilder()->BuildInsn(newMop, prevInsn->GetOperand(kInsnThirdOpnd), *labelOpnd); bb.ReplaceInsn(insn, newInsn); /* update ssa info */ ssaInfo->ReplaceInsn(insn, newInsn); @@ -503,7 +754,7 @@ void CsetCbzToBeqPattern::Run(BB &bb, Insn &insn) bool ExtLslToBitFieldInsertPattern::CheckCondition(Insn &insn) { auto &useReg = static_cast(insn.GetOperand(kInsnSecondOpnd)); - prevInsn = GetDefInsn(useReg); + prevInsn = ssaInfo->GetDefInsn(useReg); if (prevInsn == nullptr) { return false; } @@ -554,7 +805,7 @@ bool CselToCsetPattern::IsOpndDefByZero(const Insn &insn) const case MOP_wmovri32: case MOP_xmovri64: { auto &immOpnd = static_cast(insn.GetOperand(kInsnSecondOpnd)); - return immOpnd.GetValue() == 0; + return immOpnd.IsZero(); } default: return false; @@ -568,17 +819,77 @@ bool CselToCsetPattern::IsOpndDefByOne(const Insn &insn) const return false; } auto &immOpnd = static_cast(insn.GetOperand(kInsnSecondOpnd)); - return immOpnd.GetValue() == 1; + return immOpnd.IsOne(); +} + +bool CselToCsetPattern::IsOpndDefByAllOnes(const Insn &insn) const +{ + MOperator movMop = insn.GetMachineOpcode(); + if ((movMop != MOP_wmovri32) && (movMop != MOP_xmovri64)) { + return false; + } + bool is32Bits = (insn.GetOperandSize(kInsnFirstOpnd) == k32BitSize); + auto &immOpnd = static_cast(insn.GetOperand(kInsnSecondOpnd)); + return immOpnd.IsAllOnes() || (is32Bits && immOpnd.IsAllOnes32bit()); +} + +bool CselToCsetPattern::CheckZeroCondition(const Insn &insn) +{ + MOperator curMop = insn.GetMachineOpcode(); + if (curMop != MOP_wcselrrrc && curMop != MOP_xcselrrrc) { + return false; + } + RegOperand &useReg1 = static_cast(insn.GetOperand(kInsnSecondOpnd)); + RegOperand &useReg2 = static_cast(insn.GetOperand(kInsnThirdOpnd)); + if ((useReg1.GetRegisterNumber() == RZR && useReg2.GetRegisterNumber() == RZR) || + (useReg1.GetRegisterNumber() != RZR && useReg2.GetRegisterNumber() != RZR)) { + return false; + } + isZeroBefore = (useReg1.GetRegisterNumber() == RZR); + useReg = isZeroBefore ? &useReg2 : &useReg1; + if (ssaInfo) { + prevMovInsn = ssaInfo->GetDefInsn(*useReg); + } else { + prevMovInsn = insn.GetPreviousMachineInsn(); + } + if (prevMovInsn == nullptr) { + return false; + } + MOperator prevMop = prevMovInsn->GetMachineOpcode(); + if (prevMop != MOP_wmovri32 && prevMop != MOP_xmovri64) { + return false; + } + if (prevMovInsn->GetOperandSize(kInsnFirstOpnd) != insn.GetOperandSize(kInsnFirstOpnd)) { + return false; + } + if (!ssaInfo && (useReg->GetRegisterNumber() != + static_cast(prevMovInsn->GetOperand(kInsnFirstOpnd)).GetRegisterNumber())) { + return false; + } + ImmOperand &immOpnd = static_cast(prevMovInsn->GetOperand(kInsnSecondOpnd)); + isOne = immOpnd.IsOne(); + isAllOnes = + (prevMop == MOP_xmovri64 && immOpnd.IsAllOnes()) || (prevMop == MOP_wmovri32 && immOpnd.IsAllOnes32bit()); + if (!isOne && !isAllOnes) { + return false; + } + return true; } bool CselToCsetPattern::CheckCondition(Insn &insn) { + if (CheckZeroCondition(insn)) { + return true; + } + if (!ssaInfo) { + return false; + } MOperator curMop = insn.GetMachineOpcode(); if (curMop != MOP_wcselrrrc && curMop != MOP_xcselrrrc) { return false; } auto &useOpnd1 = static_cast(insn.GetOperand(kInsnSecondOpnd)); - prevMovInsn1 = GetDefInsn(useOpnd1); + prevMovInsn1 = ssaInfo->GetDefInsn(useOpnd1); if (prevMovInsn1 == nullptr) { return false; } @@ -587,7 +898,7 @@ bool CselToCsetPattern::CheckCondition(Insn &insn) return false; } auto &useOpnd2 = static_cast(insn.GetOperand(kInsnThirdOpnd)); - prevMovInsn2 = GetDefInsn(useOpnd2); + prevMovInsn2 = ssaInfo->GetDefInsn(useOpnd2); if (prevMovInsn2 == nullptr) { return false; } @@ -598,19 +909,86 @@ bool CselToCsetPattern::CheckCondition(Insn &insn) return true; } +Insn *CselToCsetPattern::BuildCondSetInsn(const Insn &cselInsn) const +{ + RegOperand &dest = static_cast(cselInsn.GetOperand(kInsnFirstOpnd)); + bool is32Bits = (cselInsn.GetOperandSize(kInsnFirstOpnd) == k32BitSize); + ConditionCode ccCode = static_cast(cselInsn.GetOperand(kInsnFourthOpnd)).GetCode(); + DEBUG_ASSERT(ccCode != kCcLast, "unknown cond, ccCode can't be kCcLast"); + AArch64CGFunc *func = static_cast(cgFunc); + Operand &rflag = func->GetOrCreateRflag(); + if (isZeroBefore) { + ConditionCode inverseCondCode = GetReverseCC(ccCode); + if (inverseCondCode == kCcLast) { + return nullptr; + } + CondOperand &cond = func->GetCondOperand(inverseCondCode); + if (isOne) { + return &cgFunc->GetInsnBuilder()->BuildInsn((is32Bits ? MOP_wcsetrc : MOP_xcsetrc), dest, cond, rflag); + } else if (isAllOnes) { + // todo: nop in BiShengC + } + } else { + CondOperand &cond = func->GetCondOperand(ccCode); + if (isOne) { + return &cgFunc->GetInsnBuilder()->BuildInsn((is32Bits ? MOP_wcsetrc : MOP_xcsetrc), dest, cond, rflag); + } else if (isAllOnes) { + // todo: nop in BiShengC + } + } + return nullptr; +} + +void CselToCsetPattern::ZeroRun(BB &bb, Insn &insn) +{ + if (!CheckCondition(insn)) { + return; + } + Insn *newInsn = BuildCondSetInsn(insn); + if (newInsn == nullptr) { + return; + } + bb.ReplaceInsn(insn, *newInsn); + if (ssaInfo) { + // update ssa info + ssaInfo->ReplaceInsn(insn, *newInsn); + } else if (static_cast(insn.GetOperand(kInsnFirstOpnd)).GetRegisterNumber() == + useReg->GetRegisterNumber()) { + bb.RemoveInsn(*prevMovInsn); + } + optSuccess = true; + SetCurrInsn(newInsn); + // dump pattern info + if (CG_PEEP_DUMP) { + std::vector prevs; + prevs.emplace_back(prevMovInsn); + DumpAfterPattern(prevs, &insn, newInsn); + } +} + void CselToCsetPattern::Run(BB &bb, Insn &insn) { if (!CheckCondition(insn)) { return; } + if (CheckZeroCondition(insn)) { + ZeroRun(bb, insn); + return; + } Operand &dstOpnd = insn.GetOperand(kInsnFirstOpnd); - MOperator newMop = (dstOpnd.GetSize() == k64BitSize ? MOP_xcsetrc : MOP_wcsetrc); + uint32 dstOpndSize = insn.GetOperandSize(kInsnFirstOpnd); + MOperator newMop = MOP_undef; Operand &condOpnd = insn.GetOperand(kInsnFourthOpnd); Operand &rflag = insn.GetOperand(kInsnFifthOpnd); Insn *newInsn = nullptr; - if (IsOpndDefByOne(*prevMovInsn1) && IsOpndDefByZero(*prevMovInsn2)) { + if (IsOpndDefByZero(*prevMovInsn2)) { + if (IsOpndDefByOne(*prevMovInsn1)) { + newMop = (dstOpndSize == k64BitSize ? MOP_xcsetrc : MOP_wcsetrc); + } else if (IsOpndDefByAllOnes(*prevMovInsn1)) { + // todo: nop in BiShengC + } newInsn = &(cgFunc->GetInsnBuilder()->BuildInsn(newMop, dstOpnd, condOpnd, rflag)); - } else if (IsOpndDefByZero(*prevMovInsn1) && IsOpndDefByOne(*prevMovInsn2)) { + } else if (IsOpndDefByZero(*prevMovInsn1)) { auto &origCondOpnd = static_cast(condOpnd); ConditionCode inverseCondCode = GetReverseCC(origCondOpnd.GetCode()); if (inverseCondCode == kCcLast) { @@ -618,9 +996,14 @@ void CselToCsetPattern::Run(BB &bb, Insn &insn) } auto *aarFunc = static_cast(cgFunc); CondOperand &inverseCondOpnd = aarFunc->GetCondOperand(inverseCondCode); + if (IsOpndDefByOne(*prevMovInsn2)) { + newMop = (dstOpndSize == k64BitSize ? MOP_xcsetrc : MOP_wcsetrc); + } else if (IsOpndDefByAllOnes(*prevMovInsn1)) { + // todo: nop in BiShengC + } newInsn = &(cgFunc->GetInsnBuilder()->BuildInsn(newMop, dstOpnd, inverseCondOpnd, rflag)); } - if (newInsn == nullptr) { + if (newMop == MOP_undef || newInsn == nullptr) { return; } bb.ReplaceInsn(insn, *newInsn); @@ -637,69 +1020,274 @@ void CselToCsetPattern::Run(BB &bb, Insn &insn) } } -bool AndCmpBranchesToTbzPattern::CheckAndSelectPattern(const Insn &currInsn) +bool CselToMovPattern::CheckCondition(Insn &insn) { - MOperator curMop = currInsn.GetMachineOpcode(); - MOperator prevAndMop = prevAndInsn->GetMachineOpcode(); - auto &andImmOpnd = static_cast(prevAndInsn->GetOperand(kInsnThirdOpnd)); - auto &cmpImmOpnd = static_cast(prevCmpInsn->GetOperand(kInsnThirdOpnd)); - if (cmpImmOpnd.GetValue() == 0) { - tbzImmVal = GetLogValueAtBase2(andImmOpnd.GetValue()); - if (tbzImmVal < 0) { - return false; - } - switch (curMop) { - case MOP_beq: - newMop = (prevAndMop == MOP_wandrri12) ? MOP_wtbz : MOP_xtbz; - break; - case MOP_bne: - newMop = (prevAndMop == MOP_wandrri12) ? MOP_wtbnz : MOP_xtbnz; - break; - default: - return false; - } - } else { - tbzImmVal = GetLogValueAtBase2(andImmOpnd.GetValue()); - int64 tmpVal = GetLogValueAtBase2(cmpImmOpnd.GetValue()); - if (tbzImmVal < 0 || tmpVal < 0 || tbzImmVal != tmpVal) { - return false; - } - switch (curMop) { - case MOP_beq: - newMop = (prevAndMop == MOP_wandrri12) ? MOP_wtbnz : MOP_xtbnz; - break; - case MOP_bne: - newMop = (prevAndMop == MOP_wandrri12) ? MOP_wtbz : MOP_xtbz; - break; - default: - return false; - } + MOperator mop = insn.GetMachineOpcode(); + if (mop != MOP_wcselrrrc && mop != MOP_xcselrrrc) { + return false; } + + if (!RegOperand::IsSameReg(insn.GetOperand(kInsnSecondOpnd), insn.GetOperand(kInsnThirdOpnd))) { + return false; + } + return true; } -bool AndCmpBranchesToTbzPattern::CheckCondition(Insn &insn) +void CselToMovPattern::Run(BB &bb, Insn &insn) { - MOperator curMop = insn.GetMachineOpcode(); - if (curMop != MOP_beq && curMop != MOP_bne) { - return false; - } - auto &ccReg = static_cast(insn.GetOperand(kInsnFirstOpnd)); - prevCmpInsn = GetDefInsn(ccReg); - if (prevCmpInsn == nullptr) { - return false; + if (!CheckCondition(insn)) { + return; } - MOperator prevCmpMop = prevCmpInsn->GetMachineOpcode(); - if (prevCmpMop != MOP_wcmpri && prevCmpMop != MOP_xcmpri) { + + MOperator newMop = insn.GetMachineOpcode() == MOP_wcselrrrc ? MOP_wmovrr : MOP_xmovrr; + Insn &newInsn = + cgFunc->GetInsnBuilder()->BuildInsn(newMop, insn.GetOperand(kInsnFirstOpnd), insn.GetOperand(kInsnSecondOpnd)); + + bb.ReplaceInsn(insn, newInsn); +} + +bool CselToCsincRemoveMovPattern::IsOpndMovOneAndNewOpndOpt(const Insn &curInsn) +{ + auto &insnThirdOpnd = static_cast(curInsn.GetOperand(kInsnThirdOpnd)); + auto &insnSecondOpnd = static_cast(curInsn.GetOperand(kInsnSecondOpnd)); + auto &origCondOpnd = static_cast(curInsn.GetOperand(kInsnFourthOpnd)); + Insn *insnThirdOpndDefInsn = ssaInfo->GetDefInsn(insnThirdOpnd); + Insn *insnSecondOpndDefInsn = ssaInfo->GetDefInsn(insnSecondOpnd); + if (insnThirdOpndDefInsn == nullptr || insnSecondOpndDefInsn == nullptr) { return false; } - auto &cmpUseReg = static_cast(prevCmpInsn->GetOperand(kInsnSecondOpnd)); - prevAndInsn = GetDefInsn(cmpUseReg); - if (prevAndInsn == nullptr) { + MOperator insnThirdOpndDefMop = insnThirdOpndDefInsn->GetMachineOpcode(); + MOperator insnSecondOpndDefMop = insnSecondOpndDefInsn->GetMachineOpcode(); + if (insnThirdOpndDefMop == MOP_wmovri32 || insnThirdOpndDefMop == MOP_xmovri64) { + prevMovInsn = insnThirdOpndDefInsn; + } else if (insnSecondOpndDefMop == MOP_wmovri32 || insnSecondOpndDefMop == MOP_xmovri64) { + prevMovInsn = insnSecondOpndDefInsn; + needReverseCond = true; + } else { return false; } - MOperator prevAndMop = prevAndInsn->GetMachineOpcode(); - if (prevAndMop != MOP_wandrri12 && prevAndMop != MOP_xandrri13) { + auto &prevMovImmOpnd = static_cast(prevMovInsn->GetOperand(kInsnSecondOpnd)); + auto val = prevMovImmOpnd.GetValue(); + if (val != 1) { + return false; + } + if (needReverseCond) { + newSecondOpnd = &insnThirdOpnd; + ConditionCode inverseCondCode = GetReverseCC(origCondOpnd.GetCode()); + if (inverseCondCode == kCcLast) { + return false; + } + auto *aarFunc = static_cast(cgFunc); + CondOperand &inverseCondOpnd = aarFunc->GetCondOperand(inverseCondCode); + cond = &inverseCondOpnd; + } else { + newSecondOpnd = &insnSecondOpnd; + cond = &origCondOpnd; + } + return true; +} + +bool CselToCsincRemoveMovPattern::CheckCondition(Insn &insn) +{ + MOperator curMop = insn.GetMachineOpcode(); + if (curMop != MOP_xcselrrrc && curMop != MOP_wcselrrrc) { + return false; + } + if (!IsOpndMovOneAndNewOpndOpt(insn)) { + return false; + } + return true; +} + +void CselToCsincRemoveMovPattern::Run(BB &bb, Insn &insn) +{ + if (!CheckCondition(insn)) { + return; + } + uint32 dstOpndSize = insn.GetOperandSize(kInsnFirstOpnd); + MOperator newMop = (dstOpndSize == k64ByteSize) ? MOP_xcsincrrrc : MOP_wcsincrrrc; + Operand &ccReg = insn.GetOperand(kInsnFifthOpnd); + RegOperand &zeroOpnd = cgFunc->GetZeroOpnd(dstOpndSize); + auto &insnFirstOpnd = static_cast(insn.GetOperand(kInsnFirstOpnd)); + Insn &newInsn = + cgFunc->GetInsnBuilder()->BuildInsn(newMop, insnFirstOpnd, *static_cast(newSecondOpnd), zeroOpnd, + *static_cast(cond), ccReg); + bb.ReplaceInsn(insn, newInsn); + // updata ssa info + ssaInfo->ReplaceInsn(insn, newInsn); + optSuccess = true; + SetCurrInsn(&newInsn); + // dump pattern info + if (CG_PEEP_DUMP) { + std::vector prevs; + prevs.emplace_back(prevMovInsn); + DumpAfterPattern(prevs, &insn, &newInsn); + } +} + +bool CsetToCincPattern::CheckDefInsn(const RegOperand &opnd, Insn &insn) +{ + Insn *tempDefInsn = ssaInfo->GetDefInsn(opnd); + if (tempDefInsn != nullptr && tempDefInsn->GetBB()->GetId() == insn.GetBB()->GetId()) { + InsnSet useInsns = GetAllUseInsn(opnd); + if (useInsns.size() != 1) { + return false; + } + MOperator mop = tempDefInsn->GetMachineOpcode(); + if (mop == MOP_wcsetrc || mop == MOP_xcsetrc) { + /* DefInsn and tempDefInsn are in the same BB. Select a close to useInsn(add) */ + if (!CheckRegTyCc(*tempDefInsn, insn)) { + return false; + } + defInsn = tempDefInsn; + return true; + } + } + return false; +} + +/* If a new ConditionCode is generated after csetInsn, this optimization is not performed. */ +bool CsetToCincPattern::CheckRegTyCc(const Insn &tempDefInsn, Insn &insn) const +{ + bool betweenUseAndDef = false; + FOR_BB_INSNS_REV(bbInsn, insn.GetBB()) + { + if (!bbInsn->IsMachineInstruction()) { + continue; + } + if (bbInsn->GetId() == insn.GetId()) { + betweenUseAndDef = true; + } + if (betweenUseAndDef) { + /* Select a close to useInsn(add) */ + if (defInsn != nullptr && bbInsn->GetId() == defInsn->GetId()) { + return false; + } else if (bbInsn->GetId() == tempDefInsn.GetId()) { + return true; + } else if (static_cast(bbInsn->GetOperand(kInsnFirstOpnd)).IsOfCC()) { + return false; + } else if (bbInsn->IsCall()) { + return false; + } + } + } + return false; +} + +bool CsetToCincPattern::CheckCondition(Insn &insn) +{ + RegOperand &opnd2 = static_cast(insn.GetOperand(kInsnSecondOpnd)); + RegOperand &opnd3 = static_cast(insn.GetOperand(kInsnThirdOpnd)); + bool opnd2Cset = CheckDefInsn(opnd2, insn); + bool opnd3Cset = CheckDefInsn(opnd3, insn); + if (opnd3Cset) { + csetOpnd1 = kInsnThirdOpnd; + return true; + } else if (opnd2Cset) { + csetOpnd1 = kInsnSecondOpnd; + return true; + } + return false; +} + +void CsetToCincPattern::Run(BB &bb, Insn &insn) +{ + RegOperand &opnd1 = static_cast(insn.GetOperand(kInsnFirstOpnd)); + /* Exclude other patterns that have been optimized. */ + Insn *newAddInsn = ssaInfo->GetDefInsn(opnd1); + if (newAddInsn == nullptr) { + return; + } + MOperator mop = newAddInsn->GetMachineOpcode(); + if (mop != MOP_waddrrr && mop != MOP_xaddrrr) { + return; + } + if (!CheckCondition(insn) || defInsn == nullptr || csetOpnd1 == 0) { + return; + } + MOperator newMop = MOP_undef; + int32 cincOpnd2 = (csetOpnd1 == kInsnSecondOpnd) ? kInsnThirdOpnd : kInsnSecondOpnd; + RegOperand &opnd2 = static_cast(insn.GetOperand(static_cast(cincOpnd2))); + Operand &condOpnd = defInsn->GetOperand(kInsnSecondOpnd); + Operand &rflag = defInsn->GetOperand(kInsnThirdOpnd); + Insn *newInsn = &(cgFunc->GetInsnBuilder()->BuildInsn(newMop, opnd1, opnd2, condOpnd, rflag)); + bb.ReplaceInsn(insn, *newInsn); + /* update ssa info */ + ssaInfo->ReplaceInsn(insn, *newInsn); + optSuccess = true; + SetCurrInsn(newInsn); + /* dump pattern info */ + if (CG_PEEP_DUMP) { + std::vector prevs; + (void)prevs.emplace_back(defInsn); + DumpAfterPattern(prevs, &insn, newInsn); + } +} + +bool AndCmpBranchesToTbzPattern::CheckAndSelectPattern(const Insn &currInsn) +{ + MOperator curMop = currInsn.GetMachineOpcode(); + MOperator prevAndMop = prevAndInsn->GetMachineOpcode(); + auto &andImmOpnd = static_cast(prevAndInsn->GetOperand(kInsnThirdOpnd)); + auto &cmpImmOpnd = static_cast(prevCmpInsn->GetOperand(kInsnThirdOpnd)); + if (cmpImmOpnd.GetValue() == 0) { + tbzImmVal = GetLogValueAtBase2(andImmOpnd.GetValue()); + if (tbzImmVal < 0) { + return false; + } + switch (curMop) { + case MOP_beq: + newMop = (prevAndMop == MOP_wandrri12) ? MOP_wtbz : MOP_xtbz; + break; + case MOP_bne: + newMop = (prevAndMop == MOP_wandrri12) ? MOP_wtbnz : MOP_xtbnz; + break; + default: + return false; + } + } else { + tbzImmVal = GetLogValueAtBase2(andImmOpnd.GetValue()); + int64 tmpVal = GetLogValueAtBase2(cmpImmOpnd.GetValue()); + if (tbzImmVal < 0 || tmpVal < 0 || tbzImmVal != tmpVal) { + return false; + } + switch (curMop) { + case MOP_beq: + newMop = (prevAndMop == MOP_wandrri12) ? MOP_wtbnz : MOP_xtbnz; + break; + case MOP_bne: + newMop = (prevAndMop == MOP_wandrri12) ? MOP_wtbz : MOP_xtbz; + break; + default: + return false; + } + } + return true; +} + +bool AndCmpBranchesToTbzPattern::CheckCondition(Insn &insn) +{ + MOperator curMop = insn.GetMachineOpcode(); + if (curMop != MOP_beq && curMop != MOP_bne) { + return false; + } + auto &ccReg = static_cast(insn.GetOperand(kInsnFirstOpnd)); + prevCmpInsn = ssaInfo->GetDefInsn(ccReg); + if (prevCmpInsn == nullptr) { + return false; + } + MOperator prevCmpMop = prevCmpInsn->GetMachineOpcode(); + if (prevCmpMop != MOP_wcmpri && prevCmpMop != MOP_xcmpri) { + return false; + } + auto &cmpUseReg = static_cast(prevCmpInsn->GetOperand(kInsnSecondOpnd)); + prevAndInsn = ssaInfo->GetDefInsn(cmpUseReg); + if (prevAndInsn == nullptr) { + return false; + } + MOperator prevAndMop = prevAndInsn->GetMachineOpcode(); + if (prevAndMop != MOP_wandrri12 && prevAndMop != MOP_xandrri13) { return false; } CHECK_FATAL(prevAndInsn->GetOperand(kInsnFirstOpnd).GetSize() == prevCmpInsn->GetOperand(kInsnSecondOpnd).GetSize(), @@ -734,6 +1322,132 @@ void AndCmpBranchesToTbzPattern::Run(BB &bb, Insn &insn) } } +bool AndAndCmpBranchesToTstPattern::CheckAndSelectPattern() +{ + MOperator prevAndMop = prevAndInsn->GetMachineOpcode(); + MOperator prevPrevAndMop = prevPrevAndInsn->GetMachineOpcode(); + if (prevAndMop != prevPrevAndMop) { + return false; + } + auto &prevAndImmOpnd = static_cast(prevAndInsn->GetOperand(kInsnThirdOpnd)); + auto &prevPrevAndImmOpnd = static_cast(prevPrevAndInsn->GetOperand(kInsnThirdOpnd)); + if (prevAndImmOpnd.GetValue() == prevPrevAndImmOpnd.GetValue() && + ((static_cast(prevAndImmOpnd.GetValue()) & static_cast(prevAndImmOpnd.GetValue() + 1)) == 0) && + ((static_cast(prevPrevAndImmOpnd.GetValue()) & + static_cast(prevPrevAndImmOpnd.GetValue() + 1)) == 0)) { + bool isWOrX = (prevAndMop == MOP_wandrri12 && prevPrevAndMop == MOP_wandrri12); + newEorMop = isWOrX ? MOP_weorrrr : MOP_xeorrrr; + newTstMop = isWOrX ? MOP_wtstri32 : MOP_xtstri64; + tstImmVal = prevAndImmOpnd.GetValue(); + return true; + } + return false; +} + +bool AndAndCmpBranchesToTstPattern::CheckCondInsn(const Insn &insn) +{ + if (insn.GetMachineOpcode() == MOP_bne || insn.GetMachineOpcode() == MOP_beq) { + ccReg = static_cast(&insn.GetOperand(kInsnFirstOpnd)); + return true; + } + if (!insn.IsCondDef()) { + return false; + } + CondOperand *condOpnd = nullptr; + for (uint32 i = 0; i < insn.GetOperandSize(); ++i) { + if (insn.GetDesc()->GetOpndDes(i) == &OpndDesc::Cond) { + condOpnd = static_cast(&insn.GetOperand(i)); + } else if (insn.GetDesc()->GetOpndDes(i) == &OpndDesc::CCS) { + ccReg = static_cast(&insn.GetOperand(i)); + } + } + if (condOpnd == nullptr || ccReg == nullptr) { + return false; + } + return (condOpnd->GetCode() == CC_NE || condOpnd->GetCode() == CC_EQ); +} + +Insn *AndAndCmpBranchesToTstPattern::CheckAndGetPrevAndDefInsn(const RegOperand ®Opnd) const +{ + if (!regOpnd.IsSSAForm()) { + return nullptr; + } + auto *regVersion = ssaInfo->FindSSAVersion(regOpnd.GetRegisterNumber()); + DEBUG_ASSERT(regVersion != nullptr, "UseVRegVersion must not be null based on ssa"); + if (regVersion->GetAllUseInsns().size() != 1) { // only one use point can do opt + return nullptr; + } + auto *defInfo = regVersion->GetDefInsnInfo(); + if (defInfo == nullptr) { + return nullptr; + } + auto andMop = defInfo->GetInsn()->GetMachineOpcode(); + if (andMop != MOP_wandrri12 && andMop != MOP_xandrri13) { + return nullptr; + } + return defInfo->GetInsn(); +} + +bool AndAndCmpBranchesToTstPattern::CheckCondition(Insn &insn) +{ + if (!CheckCondInsn(insn)) { + return false; + } + prevCmpInsn = ssaInfo->GetDefInsn(*ccReg); + if (prevCmpInsn == nullptr) { + return false; + } + MOperator prevCmpMop = prevCmpInsn->GetMachineOpcode(); + if (prevCmpMop != MOP_wcmprr && prevCmpMop != MOP_xcmprr) { + return false; + } + prevAndInsn = CheckAndGetPrevAndDefInsn(static_cast(prevCmpInsn->GetOperand(kInsnThirdOpnd))); + if (prevAndInsn == nullptr) { + return false; + } + prevPrevAndInsn = CheckAndGetPrevAndDefInsn(static_cast(prevCmpInsn->GetOperand(kInsnSecondOpnd))); + if (prevPrevAndInsn == nullptr) { + return false; + } + if (!CheckAndSelectPattern()) { + return false; + } + return true; +} + +void AndAndCmpBranchesToTstPattern ::Run(BB &bb, Insn &insn) +{ + if (!CheckCondition(insn)) { + return; + } + auto *a64Func = static_cast(cgFunc); + regno_t tmpRegNO = 0; + Operand &andOpnd = prevAndInsn->GetOperand(kInsnSecondOpnd); + auto *tmpDefOpnd = a64Func->CreateVirtualRegisterOperand(tmpRegNO, andOpnd.GetSize(), + static_cast(andOpnd).GetRegisterType()); + Insn &newEorInsn = cgFunc->GetInsnBuilder()->BuildInsn( + newEorMop, *tmpDefOpnd, prevPrevAndInsn->GetOperand(kInsnSecondOpnd), prevAndInsn->GetOperand(kInsnSecondOpnd)); + BB *preCmpBB = prevCmpInsn->GetBB(); + (void)preCmpBB->InsertInsnBefore(*prevCmpInsn, newEorInsn); + /* update ssa info */ + ssaInfo->CreateNewInsnSSAInfo(newEorInsn); + ImmOperand &tstImmOpnd = a64Func->CreateImmOperand(tstImmVal, k8BitSize, false); + Insn &newTstInsn = cgFunc->GetInsnBuilder()->BuildInsn(newTstMop, prevCmpInsn->GetOperand(kInsnFirstOpnd), + newEorInsn.GetOperand(kInsnFirstOpnd), tstImmOpnd); + bb.ReplaceInsn(*prevCmpInsn, newTstInsn); + /* update ssa info */ + ssaInfo->ReplaceInsn(*prevCmpInsn, newTstInsn); + optSuccess = true; + /* dump pattern info */ + if (CG_PEEP_DUMP) { + std::vector prevs; + prevs.emplace_back(prevPrevAndInsn); + prevs.emplace_back(prevAndInsn); + prevs.emplace_back(prevCmpInsn); + DumpAfterPattern(prevs, &newEorInsn, &newTstInsn); + } +} + bool ZeroCmpBranchesToTbzPattern::CheckAndSelectPattern(const Insn &currInsn) { MOperator currMop = currInsn.GetMachineOpcode(); @@ -819,7 +1533,7 @@ bool ZeroCmpBranchesToTbzPattern::CheckCondition(Insn &insn) } CHECK_FATAL(insn.GetOperand(kInsnSecondOpnd).IsLabel(), "must be labelOpnd"); auto &ccReg = static_cast(insn.GetOperand(kInsnFirstOpnd)); - prevInsn = GetDefInsn(ccReg); + prevInsn = ssaInfo->GetDefInsn(ccReg); if (prevInsn == nullptr) { return false; } @@ -858,6 +1572,21 @@ void ZeroCmpBranchesToTbzPattern::Run(BB &bb, Insn &insn) } } +bool LsrAndToUbfxPattern::CheckIntersectedCondition(const Insn &insn) +{ + MOperator curMop = insn.GetMachineOpcode(); + MOperator prevMop = prevInsn->GetMachineOpcode(); + int64 lsb = static_cast(prevInsn->GetOperand(kInsnThirdOpnd)).GetValue(); + int64 width = __builtin_popcountll(static_cast(insn.GetOperand(kInsnThirdOpnd)).GetValue()); + if (lsb + width <= k32BitSize) { + return true; + } else if (curMop == MOP_wandrri12 && prevMop == MOP_xlsrrri6 && lsb >= k32BitSize && (lsb + width) <= k64BitSize) { + isWXSumOutOfRange = true; + return isWXSumOutOfRange; + } + return false; +} + bool LsrAndToUbfxPattern::CheckCondition(Insn &insn) { MOperator curMop = insn.GetMachineOpcode(); @@ -870,7 +1599,7 @@ bool LsrAndToUbfxPattern::CheckCondition(Insn &insn) return false; } auto &useReg = static_cast(insn.GetOperand(kInsnSecondOpnd)); - prevInsn = GetDefInsn(useReg); + prevInsn = ssaInfo->GetDefInsn(useReg); if (prevInsn == nullptr) { return false; } @@ -878,6 +1607,11 @@ bool LsrAndToUbfxPattern::CheckCondition(Insn &insn) if (prevMop != MOP_wlsrrri5 && prevMop != MOP_xlsrrri6) { return false; } + if (((curMop == MOP_wandrri12 && prevMop == MOP_xlsrrri6) || + (curMop == MOP_xandrri13 && prevMop == MOP_wlsrrri5)) && + !CheckIntersectedCondition(insn)) { + return false; + } auto &prevDstOpnd = static_cast(prevInsn->GetOperand(kInsnFirstOpnd)); auto &currUseOpnd = static_cast(insn.GetOperand(kInsnSecondOpnd)); /* check def-use reg size found by ssa */ @@ -908,7 +1642,8 @@ void LsrAndToUbfxPattern::Run(BB &bb, Insn &insn) return; } auto *aarFunc = static_cast(cgFunc); - bool is64Bits = (static_cast(insn.GetOperand(kInsnFirstOpnd)).GetSize() == k64BitSize); + // If isWXSumOutOfRange returns true, newInsn will be 64bit + bool is64Bits = isWXSumOutOfRange ? true : (insn.GetOperandSize(kInsnFirstOpnd) == k64BitSize); Operand &resOpnd = insn.GetOperand(kInsnFirstOpnd); Operand &srcOpnd = prevInsn->GetOperand(kInsnSecondOpnd); int64 immVal1 = static_cast(prevInsn->GetOperand(kInsnThirdOpnd)).GetValue(); @@ -937,45 +1672,199 @@ void LsrAndToUbfxPattern::Run(BB &bb, Insn &insn) } } -bool MvnAndToBicPattern::CheckCondition(Insn &insn) +bool LslAndToUbfizPattern::CheckCondition(Insn &insn) { - MOperator curMop = insn.GetMachineOpcode(); - if (curMop != MOP_wandrrr && curMop != MOP_xandrrr) { + MOperator mop = insn.GetMachineOpcode(); + RegOperand &opnd2 = static_cast(insn.GetOperand(kInsnSecondOpnd)); + defInsn = ssaInfo->GetDefInsn(opnd2); + InsnSet useInsns = GetAllUseInsn(opnd2); + if (useInsns.size() != 1 || defInsn == nullptr) { return false; } - auto &useReg1 = static_cast(insn.GetOperand(kInsnSecondOpnd)); - auto &useReg2 = static_cast(insn.GetOperand(kInsnThirdOpnd)); - prevInsn1 = GetDefInsn(useReg1); - prevInsn2 = GetDefInsn(useReg2); - MOperator mop = insn.GetMachineOpcode(); - MOperator desMop = mop == MOP_xandrrr ? MOP_xnotrr : MOP_wnotrr; - op1IsMvnDef = prevInsn1 != nullptr && prevInsn1->GetMachineOpcode() == desMop; - op2IsMvnDef = prevInsn2 != nullptr && prevInsn2->GetMachineOpcode() == desMop; - if (op1IsMvnDef || op2IsMvnDef) { + MOperator defMop = defInsn->GetMachineOpcode(); + if ((mop == MOP_wandrri12 || mop == MOP_xandrri13) && (defMop == MOP_wlslrri5 || defMop == MOP_xlslrri6)) { + return true; + } else if ((defMop == MOP_wandrri12 || defMop == MOP_xandrri13) && (mop == MOP_wlslrri5 || mop == MOP_xlslrri6)) { + /* lsl w1, w2, #n. insn and w1's useInsn can do prop, skipping this pattern */ + for (auto *useInsn : GetAllUseInsn(static_cast(insn.GetOperand(kInsnFirstOpnd)))) { + if (useInsn == nullptr) { + continue; + } + if (!CheckUseInsnMop(*useInsn)) { + return false; + } + } return true; } return false; } -void MvnAndToBicPattern::Run(BB &bb, Insn &insn) +bool LslAndToUbfizPattern::CheckUseInsnMop(const Insn &useInsn) const { - if (!CheckCondition(insn)) { - return; + if (useInsn.IsLoad() || useInsn.IsStore()) { + return false; } - MOperator newMop = insn.GetMachineOpcode() == MOP_xandrrr ? MOP_xbicrrr : MOP_wbicrrr; - Insn *prevInsn = op1IsMvnDef ? prevInsn1 : prevInsn2; - auto &prevOpnd1 = static_cast(prevInsn->GetOperand(kInsnSecondOpnd)); - auto &opnd0 = static_cast(insn.GetOperand(kInsnFirstOpnd)); - auto &opnd1 = static_cast(insn.GetOperand(kInsnSecondOpnd)); - auto &opnd2 = static_cast(insn.GetOperand(kInsnThirdOpnd)); - Insn &newInsn = cgFunc->GetInsnBuilder()->BuildInsn(newMop, opnd0, op1IsMvnDef ? opnd2 : opnd1, prevOpnd1); - /* update ssa info */ - ssaInfo->ReplaceInsn(insn, newInsn); - bb.ReplaceInsn(insn, newInsn); - optSuccess = true; - SetCurrInsn(&newInsn); - /* dump pattern info */ - if (CG_PEEP_DUMP) { + MOperator useMop = useInsn.GetMachineOpcode(); + switch (useMop) { + case MOP_xeorrrr: + case MOP_xeorrrrs: + case MOP_weorrrr: + case MOP_weorrrrs: + case MOP_xiorrrr: + case MOP_xiorrrrs: + case MOP_wiorrrr: + case MOP_wiorrrrs: + case MOP_xaddrrr: + case MOP_xxwaddrrre: + case MOP_xaddrrrs: + case MOP_waddrrr: + case MOP_wwwaddrrre: + case MOP_waddrrrs: + case MOP_waddrri12: + case MOP_xaddrri12: + case MOP_xsubrrr: + case MOP_xxwsubrrre: + case MOP_xsubrrrs: + case MOP_wsubrrr: + case MOP_wwwsubrrre: + case MOP_wsubrrrs: + case MOP_xinegrr: + case MOP_winegrr: + case MOP_xsxtb32: + case MOP_xsxtb64: + case MOP_xsxth32: + case MOP_xsxth64: + case MOP_xuxtb32: + case MOP_xuxth32: + case MOP_xsxtw64: + case MOP_xubfxrri6i6: + case MOP_xcmprr: + case MOP_xwcmprre: + case MOP_xcmprrs: + case MOP_wcmprr: + case MOP_wwcmprre: + case MOP_wcmprrs: + return false; + default: + break; + } + return true; +} + +void LslAndToUbfizPattern::Run(BB &bb, Insn &insn) +{ + if (!CheckCondition(insn)) { + return; + } + MOperator mop = insn.GetMachineOpcode(); + Insn *newInsn = nullptr; + if (mop == MOP_wandrri12 || mop == MOP_xandrri13) { + newInsn = BuildNewInsn(insn, *defInsn, insn); + } + if (mop == MOP_wlslrri5 || mop == MOP_xlslrri6) { + newInsn = BuildNewInsn(*defInsn, insn, insn); + } + if (newInsn == nullptr) { + return; + } + bb.ReplaceInsn(insn, *newInsn); + /* update ssa info */ + ssaInfo->ReplaceInsn(insn, *newInsn); + optSuccess = true; + SetCurrInsn(newInsn); + /* dump pattern info */ + if (CG_PEEP_DUMP) { + std::vector prevs; + (void)prevs.emplace_back(defInsn); + DumpAfterPattern(prevs, &insn, newInsn); + } +} + +// Build ubfiz insn or mov insn +Insn *LslAndToUbfizPattern::BuildNewInsn(const Insn &andInsn, const Insn &lslInsn, const Insn &useInsn) const +{ + uint64 andImmValue = static_cast(static_cast(andInsn.GetOperand(kInsnThirdOpnd)).GetValue()); + uint64 lslImmValue = static_cast(static_cast(lslInsn.GetOperand(kInsnThirdOpnd)).GetValue()); + MOperator useMop = useInsn.GetMachineOpcode(); + // isLslAnd means true -> lsl + and, false -> and + lsl + bool isLslAnd = (useMop == MOP_wandrri12) || (useMop == MOP_xandrri13); + // judgment need to set non-zero value + uint64 judgment = 1; + // When useInsn is lsl, check whether the value of immValue is 2^n-1. + // When useInsn is and, check whether the value of immValue is (2^n-1) << m + if (isLslAnd) { + if ((andImmValue >> lslImmValue) != 0) { + judgment = (andImmValue >> lslImmValue) & ((andImmValue >> lslImmValue) + 1); + } + } else { + judgment = andImmValue & (andImmValue + 1); + } + if (judgment != 0) { + return nullptr; + } + RegOperand &ubfizOpnd1 = static_cast(useInsn.GetOperand(kInsnFirstOpnd)); + uint32 opnd1Size = ubfizOpnd1.GetSize(); + RegOperand &ubfizOpnd2 = static_cast(defInsn->GetOperand(kInsnSecondOpnd)); + uint32 opnd2Size = ubfizOpnd2.GetSize(); + ImmOperand &ubfizOpnd3 = static_cast(lslInsn.GetOperand(kInsnThirdOpnd)); + uint32 mValue = static_cast(ubfizOpnd3.GetValue()); + uint32 nValue = 0; + if (isLslAnd) { + nValue = static_cast(__builtin_popcountll(andImmValue >> lslImmValue)); + } else { + nValue = static_cast(__builtin_popcountll(andImmValue)); + } + auto *aarFunc = static_cast(cgFunc); + if (opnd1Size != opnd2Size || (mValue + nValue) > opnd1Size) { + return nullptr; + } + MOperator addMop = andInsn.GetMachineOpcode(); + MOperator newMop = (addMop == MOP_wandrri12) ? MOP_wubfizrri5i5 : MOP_xubfizrri6i6; + uint32 size = (addMop == MOP_wandrri12) ? kMaxImmVal5Bits : kMaxImmVal6Bits; + ImmOperand &ubfizOpnd4 = aarFunc->CreateImmOperand(nValue, size, false); + Insn &newInsn = cgFunc->GetInsnBuilder()->BuildInsn(newMop, ubfizOpnd1, ubfizOpnd2, ubfizOpnd3, ubfizOpnd4); + return &newInsn; +} + +bool MvnAndToBicPattern::CheckCondition(Insn &insn) +{ + MOperator curMop = insn.GetMachineOpcode(); + if (curMop != MOP_wandrrr && curMop != MOP_xandrrr) { + return false; + } + auto &useReg1 = static_cast(insn.GetOperand(kInsnSecondOpnd)); + auto &useReg2 = static_cast(insn.GetOperand(kInsnThirdOpnd)); + prevInsn1 = ssaInfo->GetDefInsn(useReg1); + prevInsn2 = ssaInfo->GetDefInsn(useReg2); + MOperator mop = insn.GetMachineOpcode(); + MOperator desMop = mop == MOP_xandrrr ? MOP_xnotrr : MOP_wnotrr; + op1IsMvnDef = prevInsn1 != nullptr && prevInsn1->GetMachineOpcode() == desMop; + op2IsMvnDef = prevInsn2 != nullptr && prevInsn2->GetMachineOpcode() == desMop; + if (op1IsMvnDef || op2IsMvnDef) { + return true; + } + return false; +} + +void MvnAndToBicPattern::Run(BB &bb, Insn &insn) +{ + if (!CheckCondition(insn)) { + return; + } + MOperator newMop = insn.GetMachineOpcode() == MOP_xandrrr ? MOP_xbicrrr : MOP_wbicrrr; + Insn *prevInsn = op1IsMvnDef ? prevInsn1 : prevInsn2; + auto &prevOpnd1 = static_cast(prevInsn->GetOperand(kInsnSecondOpnd)); + auto &opnd0 = static_cast(insn.GetOperand(kInsnFirstOpnd)); + auto &opnd1 = static_cast(insn.GetOperand(kInsnSecondOpnd)); + auto &opnd2 = static_cast(insn.GetOperand(kInsnThirdOpnd)); + Insn &newInsn = cgFunc->GetInsnBuilder()->BuildInsn(newMop, opnd0, op1IsMvnDef ? opnd2 : opnd1, prevOpnd1); + /* update ssa info */ + ssaInfo->ReplaceInsn(insn, newInsn); + bb.ReplaceInsn(insn, newInsn); + optSuccess = true; + SetCurrInsn(&newInsn); + /* dump pattern info */ + if (CG_PEEP_DUMP) { std::vector prevs; prevs.emplace_back(prevInsn); DumpAfterPattern(prevs, &insn, &newInsn); @@ -989,7 +1878,7 @@ bool AndCbzToTbzPattern::CheckCondition(Insn &insn) return false; } auto &useReg = static_cast(insn.GetOperand(kInsnFirstOpnd)); - prevInsn = ssaInfo ? GetDefInsn(useReg) : insn.GetPreviousMachineInsn(); + prevInsn = ssaInfo ? ssaInfo->GetDefInsn(useReg) : insn.GetPreviousMachineInsn(); if (prevInsn == nullptr) { return false; } @@ -1035,16 +1924,97 @@ void AndCbzToTbzPattern::Run(BB &bb, Insn &insn) } auto &labelOpnd = static_cast(insn.GetOperand(kInsnSecondOpnd)); ImmOperand &tbzImm = aarchFunc->CreateImmOperand(tbzVal, k8BitSize, false); - Insn &newInsn = - cgFunc->GetInsnBuilder()->BuildInsn(newMop, prevInsn->GetOperand(kInsnSecondOpnd), tbzImm, labelOpnd); - bb.ReplaceInsn(insn, newInsn); + Insn *newInsn = nullptr; + // if bit offset is invalid (implicit zero), then it should be a unconditional branch + if (!aarchFunc->IsOperandImmValid(newMop, &tbzImm, kInsnSecondOpnd)) { + // cfg adjustment in ssa is complicate, so we just bypass this pattern if imm is invalid. + if (ssaInfo) { + return; + } + bool delEdgeWithTarget = false; + if (newMop == MOP_wtbz) { + newInsn = &aarchFunc->GetInsnBuilder()->BuildInsn(MOP_xuncond, labelOpnd); + bb.SetKind(BB::kBBGoto); + } else if (newMop == MOP_wtbnz) { + bb.SetKind(BB::kBBFallthru); + bb.RemoveInsn(insn); + delEdgeWithTarget = true; + } else { + CHECK_FATAL(false, "only wtbz/wtbnz can have invalid imm"); + } + auto *bbFt = bb.GetNext(); + auto *targetBB = cgFunc->GetBBFromLab2BBMap(labelOpnd.GetLabelIndex()); + if (targetBB != bbFt) { // when targetBB is ftBB, we cannot remove preds/succs + auto *delEdgeBB = delEdgeWithTarget ? targetBB : bbFt; + delEdgeBB->RemovePreds(bb); + bb.RemoveSuccs(*delEdgeBB); + } + } else { + newInsn = + &cgFunc->GetInsnBuilder()->BuildInsn(newMop, prevInsn->GetOperand(kInsnSecondOpnd), tbzImm, labelOpnd); + } + // try opt failed + if (newInsn == nullptr) { + return; + } + bb.ReplaceInsn(insn, *newInsn); + SetCurrInsn(newInsn); if (ssaInfo) { /* update ssa info */ - ssaInfo->ReplaceInsn(insn, newInsn); + ssaInfo->ReplaceInsn(insn, *newInsn); } optSuccess = true; - SetCurrInsn(&newInsn); /* dump pattern info */ + if (CG_PEEP_DUMP) { + std::vector prevs; + prevs.emplace_back(prevInsn); + DumpAfterPattern(prevs, &insn, newInsn); + } +} + +bool AndTbzPattern::CheckCondition(Insn &insn) +{ + MOperator curMop = insn.GetMachineOpcode(); + if (curMop != MOP_wtbz && curMop != MOP_xtbz && curMop != MOP_wtbnz && curMop != MOP_xtbnz) { + return false; + } + auto &useReg = static_cast(insn.GetOperand(kInsnFirstOpnd)); + prevInsn = ssaInfo ? ssaInfo->GetDefInsn(useReg) : insn.GetPreviousMachineInsn(); + if (prevInsn == nullptr) { + return false; + } + MOperator prevMop = prevInsn->GetMachineOpcode(); + if (prevMop != MOP_wandrri12 && prevMop != MOP_xandrri13) { + return false; + } + if (!ssaInfo && (&(prevInsn->GetOperand(kInsnFirstOpnd)) != &(insn.GetOperand(kInsnFirstOpnd)))) { + return false; + } + return true; +} + +void AndTbzPattern::Run(BB &bb, Insn &insn) +{ + if (!CheckCondition(insn)) { + return; + } + auto tbzVal = static_cast(insn.GetOperand(kInsnSecondOpnd)).GetValue(); + auto andVal = static_cast(prevInsn->GetOperand(kInsnThirdOpnd)).GetValue(); + CHECK_FATAL(tbzVal >= 0 && tbzVal < k64BitSize, "NIY, tbz imm val out of range."); + if ((static_cast(andVal) & (1UL << tbzVal)) == 0) { + return; + } + auto &newOpnd = prevInsn->GetOperand(kInsnSecondOpnd); + auto &newInsn = cgFunc->GetInsnBuilder()->BuildInsn( + insn.GetMachineOpcode(), newOpnd, insn.GetOperand(kInsnSecondOpnd), insn.GetOperand(kInsnThirdOpnd)); + bb.ReplaceInsn(insn, newInsn); + SetCurrInsn(&newInsn); + if (ssaInfo) { + // update ssa info + ssaInfo->ReplaceInsn(insn, newInsn); + } + optSuccess = true; + // dump pattern info if (CG_PEEP_DUMP) { std::vector prevs; prevs.emplace_back(prevInsn); @@ -1060,7 +2030,7 @@ bool CombineSameArithmeticPattern::CheckCondition(Insn &insn) } Operand &useOpnd = insn.GetOperand(kInsnSecondOpnd); CHECK_FATAL(useOpnd.IsRegister(), "expect regOpnd"); - prevInsn = GetDefInsn(static_cast(useOpnd)); + prevInsn = ssaInfo->GetDefInsn(static_cast(useOpnd)); if (prevInsn == nullptr) { return false; } @@ -1082,6 +2052,16 @@ bool CombineSameArithmeticPattern::CheckCondition(Insn &insn) int64 curImm = curImmOpnd.GetValue(); newImmOpnd = &aarFunc->CreateImmOperand(prevImmOpnd.GetValue() + curImmOpnd.GetValue(), curImmOpnd.GetSize(), curImmOpnd.IsSignedValue()); + // prop vary attr + if (prevImmOpnd.GetVary() == kUnAdjustVary && curImmOpnd.GetVary() == kUnAdjustVary) { + return false; + } + if (prevImmOpnd.GetVary() == kUnAdjustVary || curImmOpnd.GetVary() == kUnAdjustVary) { + newImmOpnd->SetVary(kUnAdjustVary); + } + if (prevImmOpnd.GetVary() == kAdjustVary || curImmOpnd.GetVary() == kAdjustVary) { + newImmOpnd->SetVary(kAdjustVary); + } switch (curMop) { case MOP_wlsrrri5: case MOP_wasrrri5: @@ -1144,9 +2124,9 @@ bool LogicShiftAndOrrToExtrPattern::CheckCondition(Insn &insn) is64Bits = (curDstOpnd.GetSize() == k64BitSize); if (curMop == MOP_wiorrrr || curMop == MOP_xiorrrr) { auto &useReg1 = static_cast(insn.GetOperand(kInsnSecondOpnd)); - Insn *prevInsn1 = GetDefInsn(useReg1); + Insn *prevInsn1 = ssaInfo->GetDefInsn(useReg1); auto &useReg2 = static_cast(insn.GetOperand(kInsnThirdOpnd)); - Insn *prevInsn2 = GetDefInsn(useReg2); + Insn *prevInsn2 = ssaInfo->GetDefInsn(useReg2); if (prevInsn1 == nullptr || prevInsn2 == nullptr) { return false; } @@ -1175,7 +2155,7 @@ bool LogicShiftAndOrrToExtrPattern::CheckCondition(Insn &insn) shiftValue = prevLsrImmValue; } else if (curMop == MOP_wiorrrrs || curMop == MOP_xiorrrrs) { auto &useReg = static_cast(insn.GetOperand(kInsnSecondOpnd)); - Insn *prevInsn = GetDefInsn(useReg); + Insn *prevInsn = ssaInfo->GetDefInsn(useReg); if (prevInsn == nullptr) { return false; } @@ -1295,12 +2275,13 @@ bool SimplifyMulArithmeticPattern::CheckCondition(Insn &insn) return false; } auto &useReg = static_cast(insn.GetOperand(static_cast(validOpndIdx))); - prevInsn = GetDefInsn(useReg); + prevInsn = ssaInfo->GetDefInsn(useReg); if (prevInsn == nullptr) { return false; } regno_t useRegNO = useReg.GetRegisterNumber(); VRegVersion *useVersion = ssaInfo->FindSSAVersion(useRegNO); + DEBUG_ASSERT(useVersion != nullptr, "useVersion should not be nullptr"); if (useVersion->GetAllUseInsns().size() > 1) { return false; } @@ -1419,6 +2400,16 @@ void ElimSpecificExtensionPattern::SetSpecificExtType(const Insn &currInsn) extTypeIdx = UXTW; break; } + case MOP_wandrri12: { + is64Bits = false; + extTypeIdx = AND; + break; + } + case MOP_xandrri13: { + is64Bits = true; + extTypeIdx = AND; + break; + } default: { extTypeIdx = EXTUNDEF; } @@ -1468,23 +2459,44 @@ void ElimSpecificExtensionPattern::ReplaceExtWithMov(Insn &currInsn) { auto &prevDstOpnd = static_cast(prevInsn->GetOperand(kInsnFirstOpnd)); auto &currDstOpnd = static_cast(currInsn.GetOperand(kInsnFirstOpnd)); - MOperator newMop = is64Bits ? MOP_xmovrr : MOP_wmovrr; - Insn &newInsn = cgFunc->GetInsnBuilder()->BuildInsn(newMop, currDstOpnd, prevDstOpnd); - currBB->ReplaceInsn(currInsn, newInsn); - /* update ssa info */ - ssaInfo->ReplaceInsn(currInsn, newInsn); + bool isInsnDeleted = false; + if (sceneType == kSceneSameExt && currDstOpnd.IsSSAForm() && prevDstOpnd.IsSSAForm()) { + auto *destVersion = ssaInfo->FindSSAVersion(currDstOpnd.GetRegisterNumber()); + DEBUG_ASSERT(destVersion != nullptr, "find Version failed"); + auto *srcVersion = ssaInfo->FindSSAVersion(prevDstOpnd.GetRegisterNumber()); + DEBUG_ASSERT(srcVersion != nullptr, "find Version failed"); + ssaInfo->ReplaceAllUse(destVersion, srcVersion); + if (destVersion->GetAllUseInsns().empty()) { + currInsn.GetBB()->RemoveInsn(currInsn); + destVersion->MarkDeleted(); + MapleUnorderedMap &useInfos = srcVersion->GetAllUseInsns(); + auto it = useInfos.find(currInsn.GetId()); + if (it != useInfos.end()) { + useInfos.erase(it); + } + isInsnDeleted = true; + } + } + Insn *newInsn = nullptr; + if (!isInsnDeleted) { + MOperator newMop = is64Bits ? MOP_xmovrr : MOP_wmovrr; + newInsn = &cgFunc->GetInsnBuilder()->BuildInsn(newMop, currDstOpnd, prevDstOpnd); + currBB->ReplaceInsn(currInsn, *newInsn); + /* update ssa info */ + ssaInfo->ReplaceInsn(currInsn, *newInsn); + } optSuccess = true; /* dump pattern info */ if (CG_PEEP_DUMP) { std::vector prevs; prevs.emplace_back(prevInsn); - DumpAfterPattern(prevs, &currInsn, &newInsn); + DumpAfterPattern(prevs, &currInsn, newInsn); } } void ElimSpecificExtensionPattern::ElimExtensionAfterMov(Insn &insn) { - if (&insn == currBB->GetFirstInsn()) { + if (&insn == currBB->GetFirstMachineInsn() || extTypeIdx == AND) { return; } auto &prevDstOpnd = static_cast(prevInsn->GetOperand(kInsnFirstOpnd)); @@ -1519,26 +2531,28 @@ void ElimSpecificExtensionPattern::ElimExtensionAfterMov(Insn &insn) ReplaceExtWithMov(insn); } } else if (currMop == MOP_xuxtb32 || currMop == MOP_xuxth32) { - if (!(static_cast(value) & minRange)) { + if ((static_cast(value) & minRange) == 0) { ReplaceExtWithMov(insn); } } else if (currMop == MOP_xuxtw64) { ReplaceExtWithMov(insn); } else { /* MOP_xsxtb64 & MOP_xsxth64 & MOP_xsxtw64 */ - if (!(static_cast(value) & minRange) && immMovOpnd.IsSingleInstructionMovable(currDstOpnd.GetSize())) { + if ((static_cast(value) & minRange) == 0 && + immMovOpnd.IsSingleInstructionMovable(currDstOpnd.GetSize())) { ReplaceExtWithMov(insn); } } } -bool ElimSpecificExtensionPattern::IsValidLoadExtPattern(Insn &currInsn, MOperator oldMop, MOperator newMop) const +bool ElimSpecificExtensionPattern::IsValidLoadExtPattern(MOperator oldMop, MOperator newMop) const { if (oldMop == newMop) { return true; } auto *aarFunc = static_cast(cgFunc); auto *memOpnd = static_cast(prevInsn->GetMemOpnd()); + CHECK_FATAL(memOpnd != nullptr, "invalid memOpnd"); DEBUG_ASSERT(!prevInsn->IsStorePair(), "do not do ElimSpecificExtensionPattern for str pair"); DEBUG_ASSERT(!prevInsn->IsLoadPair(), "do not do ElimSpecificExtensionPattern for ldr pair"); if (memOpnd->GetAddrMode() == MemOperand::kAddrModeBOi && @@ -1584,14 +2598,20 @@ void ElimSpecificExtensionPattern::ElimExtensionAfterLoad(Insn &insn) if (extTypeIdx == EXTUNDEF) { return; } + if (extTypeIdx == AND) { + auto &immOpnd = static_cast(insn.GetOperand(kInsnThirdOpnd)); + if (immOpnd.GetValue() != 0xff) { + return; + } + } MOperator prevOrigMop = prevInsn->GetMachineOpcode(); for (uint8 i = 0; i < kPrevLoadPatternNum; i++) { - DEBUG_ASSERT(extTypeIdx < SpecificExtTypeSize, "extTypeIdx must be lower than SpecificExtTypeSize"); + DEBUG_ASSERT(extTypeIdx < EXTTYPESIZE, "extTypeIdx must be lower than EXTTYPESIZE"); if (prevOrigMop != loadMappingTable[extTypeIdx][i][0]) { continue; } MOperator prevNewMop = loadMappingTable[extTypeIdx][i][1]; - if (!IsValidLoadExtPattern(insn, prevOrigMop, prevNewMop)) { + if (!IsValidLoadExtPattern(prevOrigMop, prevNewMop)) { return; } if (is64Bits && extTypeIdx >= SXTB && extTypeIdx <= SXTW) { @@ -1650,7 +2670,7 @@ void ElimSpecificExtensionPattern::ElimExtensionAfterLoad(Insn &insn) void ElimSpecificExtensionPattern::ElimExtensionAfterSameExt(Insn &insn) { - if (extTypeIdx == EXTUNDEF) { + if (extTypeIdx == EXTUNDEF || extTypeIdx == AND) { return; } auto &prevDstOpnd = static_cast(prevInsn->GetOperand(kInsnFirstOpnd)); @@ -1661,7 +2681,7 @@ void ElimSpecificExtensionPattern::ElimExtensionAfterSameExt(Insn &insn) MOperator prevMop = prevInsn->GetMachineOpcode(); MOperator currMop = insn.GetMachineOpcode(); for (uint8 i = 0; i < kSameExtPatternNum; i++) { - DEBUG_ASSERT(extTypeIdx < SpecificExtTypeSize, "extTypeIdx must be lower than SpecificExtTypeSize"); + DEBUG_ASSERT(extTypeIdx < EXTTYPESIZE, "extTypeIdx must be lower than EXTTYPESIZE"); if (sameExtMappingTable[extTypeIdx][i][0] == MOP_undef || sameExtMappingTable[extTypeIdx][i][1] == MOP_undef) { continue; } @@ -1674,7 +2694,7 @@ void ElimSpecificExtensionPattern::ElimExtensionAfterSameExt(Insn &insn) bool ElimSpecificExtensionPattern::CheckCondition(Insn &insn) { auto &useReg = static_cast(insn.GetOperand(kInsnSecondOpnd)); - prevInsn = GetDefInsn(useReg); + prevInsn = ssaInfo->GetDefInsn(useReg); InsnSet useInsns = GetAllUseInsn(useReg); if ((prevInsn == nullptr) || (useInsns.size() != 1)) { return false; @@ -1687,7 +2707,7 @@ bool ElimSpecificExtensionPattern::CheckCondition(Insn &insn) return true; } -void ElimSpecificExtensionPattern::Run(BB &bb, Insn &insn) +void ElimSpecificExtensionPattern::Run(BB & /* bb */, Insn &insn) { if (!CheckCondition(insn)) { return; @@ -1703,7 +2723,7 @@ void ElimSpecificExtensionPattern::Run(BB &bb, Insn &insn) void OneHoleBranchPattern::FindNewMop(const BB &bb, const Insn &insn) { - if (&insn != bb.GetLastInsn()) { + if (&insn != bb.GetLastMachineInsn()) { return; } MOperator thisMop = insn.GetMachineOpcode(); @@ -1795,7 +2815,7 @@ bool OneHoleBranchPattern::CheckCondition(Insn &insn) return false; } auto &useReg = static_cast(insn.GetOperand(kInsnFirstOpnd)); - prevInsn = GetDefInsn(useReg); + prevInsn = ssaInfo->GetDefInsn(useReg); if (prevInsn == nullptr) { return false; } @@ -1808,7 +2828,7 @@ bool OneHoleBranchPattern::CheckCondition(Insn &insn) bool OneHoleBranchPattern::CheckPrePrevInsn() { auto &useReg = static_cast(prevInsn->GetOperand(kInsnSecondOpnd)); - prePrevInsn = GetDefInsn(useReg); + prePrevInsn = ssaInfo->GetDefInsn(useReg); if (prePrevInsn == nullptr) { return false; } @@ -1905,7 +2925,9 @@ void AArch64CGPeepHole::DoNormalOptimize(BB &bb, Insn &insn) break; } case MOP_wstrb: - case MOP_wldrb: + case MOP_wldrb: { + // only strb ldrb can do this pattern, other patterns still need to be done, so there is no break here. + } case MOP_wstrh: case MOP_wldrh: case MOP_xldr: @@ -1919,8 +2941,12 @@ void AArch64CGPeepHole::DoNormalOptimize(BB &bb, Insn &insn) case MOP_qldr: case MOP_qstr: { manager->NormalPatternOpt(cgFunc->IsAfterRegAlloc()); - manager->NormalPatternOpt(cgFunc->IsAfterRegAlloc()); - manager->NormalPatternOpt(cgFunc->IsAfterRegAlloc()); + if (!manager->OptSuccess()) { + manager->NormalPatternOpt(cgFunc->IsAfterRegAlloc()); + } + if (!manager->OptSuccess()) { + manager->NormalPatternOpt(cgFunc->IsAfterRegAlloc()); + } break; } case MOP_xvmovrv: @@ -1937,7 +2963,9 @@ void AArch64CGPeepHole::DoNormalOptimize(BB &bb, Insn &insn) case MOP_wcbnz: case MOP_xcbnz: { manager->NormalPatternOpt(!cgFunc->IsAfterRegAlloc()); - manager->NormalPatternOpt(cgFunc->IsAfterRegAlloc()); + if (!manager->OptSuccess()) { + manager->NormalPatternOpt(cgFunc->IsAfterRegAlloc()); + } break; } case MOP_wsdivrrr: { @@ -2102,7 +3130,7 @@ void AArch64PeepHole0::Run(BB &bb, Insn &insn) (static_cast(optimizations[kRemoveIdenticalLoadAndStoreOpt])) ->Run(bb, insn); } - (static_cast(optimizations[kEnhanceStrLdrAArch64Opt]))->Run(bb, insn); + (static_cast(optimizations[kEnhanceStrLdrAArch64Opt]))->Run(bb, insn); break; } default: @@ -2224,86 +3252,188 @@ void AArch64PrePeepHole1::Run(BB &bb, Insn &insn) } } -bool RemoveIdenticalLoadAndStorePattern::CheckCondition(Insn &insn) +bool RemoveIdenticalLoadAndStorePattern::IsIdenticalMemOpcode(const Insn &curInsn, const Insn &checkedInsn) const { - nextInsn = insn.GetNextMachineInsn(); - if (nextInsn == nullptr) { - return false; + if (checkedInsn.GetMachineOpcode() == curInsn.GetMachineOpcode()) { + return true; } - return true; + if (curInsn.IsLoad() && checkedInsn.IsStore()) { + MOperator curMop = curInsn.GetMachineOpcode(); + MOperator checkedMop = checkedInsn.GetMachineOpcode(); + // We do not eliminate wldr after wstr, to ensure that the high bits are cleared + if (curMop == MOP_xldr && checkedMop == MOP_xstr) { + return true; + } + if (curMop == MOP_dldr && checkedMop == MOP_dstr) { + return true; + } + if (curMop == MOP_sldr && checkedMop == MOP_sstr) { + return true; + } + if (curMop == MOP_qldr && checkedMop == MOP_qstr) { + return true; + } + } + // For str[BOI] + str[BOI]: they do not need to have same [srcOpnd], as long as the size of srcOpnds are same + if (checkedInsn.IsStore() && curInsn.IsStore() && checkedInsn.GetMemoryByteSize() == curInsn.GetMemoryByteSize()) { + return true; + } + return false; } -void RemoveIdenticalLoadAndStorePattern::Run(BB &bb, Insn &insn) +Insn *RemoveIdenticalLoadAndStorePattern::FindPrevIdenticalMemInsn(const Insn &curInsn) const { - if (!CheckCondition(insn)) { - return; + auto *curMemOpnd = static_cast(curInsn.GetMemOpnd()); + ASSERT_NOT_NULL(curMemOpnd); + RegOperand *curBaseOpnd = curMemOpnd->GetBaseRegister(); + OfstOperand *curOfstOpnd = curMemOpnd->GetOffsetImmediate(); + auto &curFirstOpnd = static_cast(curInsn.GetOperand(kInsnFirstOpnd)); + for (Insn *checkedInsn = curInsn.GetPreviousMachineInsn(); checkedInsn != nullptr; + checkedInsn = checkedInsn->GetPreviousMachineInsn()) { + if (checkedInsn->IsCall() || checkedInsn->IsSpecialCall()) { + return nullptr; + } + if (checkedInsn->IsRegDefined(curBaseOpnd->GetRegisterNumber())) { + return nullptr; + } + if (curInsn.IsLoad() && checkedInsn->IsRegDefined(curFirstOpnd.GetRegisterNumber())) { + return nullptr; + } + if (!checkedInsn->IsStore() && !checkedInsn->IsLoad()) { + continue; + } + if (!IsIdenticalMemOpcode(curInsn, *checkedInsn)) { + continue; + } + auto *checkedMemOpnd = static_cast(checkedInsn->GetMemOpnd()); + CHECK_FATAL(checkedMemOpnd != nullptr, "invalid mem instruction"); + if (checkedMemOpnd->GetAddrMode() != MemOperand::kAddrModeBOi || checkedMemOpnd->GetBaseRegister() == nullptr || + checkedMemOpnd->GetOffsetImmediate() == nullptr) { + continue; + } + RegOperand *checkedBaseOpnd = checkedMemOpnd->GetBaseRegister(); + OfstOperand *checkedOfstOpnd = checkedMemOpnd->GetOffsetImmediate(); + auto &checkedFirstOpnd = static_cast(checkedInsn->GetOperand(kInsnFirstOpnd)); + if (checkedBaseOpnd->GetRegisterNumber() == curBaseOpnd->GetRegisterNumber() && + checkedOfstOpnd->GetValue() == curOfstOpnd->GetValue() && + (checkedFirstOpnd.GetRegisterNumber() == curFirstOpnd.GetRegisterNumber() || curInsn.IsStore())) { + return checkedInsn; + } } - MOperator mop1 = insn.GetMachineOpcode(); - MOperator mop2 = nextInsn->GetMachineOpcode(); - if ((mop1 == MOP_wstr && mop2 == MOP_wstr) || (mop1 == MOP_xstr && mop2 == MOP_xstr)) { - if (IsMemOperandsIdentical(insn, *nextInsn)) { - bb.RemoveInsn(insn); + return nullptr; +} + +bool RemoveIdenticalLoadAndStorePattern::HasMemReferenceBetweenTwoInsns(const Insn &curInsn) const +{ + auto *curMemOpnd = static_cast(curInsn.GetMemOpnd()); + RegOperand *curBaseOpnd = curMemOpnd->GetBaseRegister(); + OfstOperand *curOfstOpnd = curMemOpnd->GetOffsetImmediate(); + for (Insn *checkedInsn = curInsn.GetPreviousMachineInsn(); + checkedInsn != prevIdenticalInsn && checkedInsn != nullptr; + checkedInsn = checkedInsn->GetPreviousMachineInsn()) { + // Check mem alias + if ((checkedInsn->IsMemAccess() || checkedInsn->IsMemAccessBar()) && + AArch64MemReference::HasAliasMemoryDep(*checkedInsn, curInsn, kDependenceTypeNone)) { + return true; } - } else if ((mop1 == MOP_wstr && mop2 == MOP_wldr) || (mop1 == MOP_xstr && mop2 == MOP_xldr)) { - if (IsMemOperandsIdentical(insn, *nextInsn)) { - bb.RemoveInsn(*nextInsn); + auto *checkedMemOpnd = static_cast(checkedInsn->GetMemOpnd()); + if (checkedMemOpnd == nullptr) { + continue; + } + RegOperand *checkedBaseOpnd = checkedMemOpnd->GetBaseRegister(); + OfstOperand *checkedOfstOpnd = checkedMemOpnd->GetOffsetImmediate(); + if (checkedBaseOpnd == nullptr || checkedOfstOpnd == nullptr) { + continue; + } + if (checkedBaseOpnd->GetRegisterNumber() == curBaseOpnd->GetRegisterNumber()) { + // Check mem overlap + int64 curBaseOfst = curOfstOpnd->GetValue(); + int64 checkedOfst = checkedOfstOpnd->GetValue(); + auto curMemRange = static_cast(curInsn.GetMemoryByteSize()); + auto checkedMemRange = static_cast(checkedInsn->GetMemoryByteSize()); + if ((curBaseOfst >= checkedOfst && curBaseOfst < (checkedOfst + checkedMemRange)) || + (checkedOfst >= curBaseOfst && checkedOfst < (curBaseOfst + curMemRange))) { + return true; + } } } + return false; } -bool RemoveIdenticalLoadAndStorePattern::IsMemOperandsIdentical(const Insn &insn1, const Insn &insn2) const +bool RemoveIdenticalLoadAndStorePattern::HasImplictSizeUse(const Insn &curInsn) const { - regno_t regNO1 = static_cast(insn1.GetOperand(kInsnFirstOpnd)).GetRegisterNumber(); - regno_t regNO2 = static_cast(insn2.GetOperand(kInsnFirstOpnd)).GetRegisterNumber(); - if (regNO1 != regNO2) { - return false; + if (prevIdenticalInsn->GetOperandSize(kInsnFirstOpnd) != curInsn.GetOperandSize(kInsnFirstOpnd)) { + return true; } - /* Match only [base + offset] */ - auto &memOpnd1 = static_cast(insn1.GetOperand(kInsnSecondOpnd)); - if (memOpnd1.GetAddrMode() != MemOperand::kAddrModeBOi || !memOpnd1.IsIntactIndexed()) { - return false; + // To avoid the optimization as following: + // str w10, [sp, #8] + // ldr w10, [sp, #8] ---\--> can not be removed + // ... + // str x10, [x1, #16] + if (curInsn.IsLoad() && prevIdenticalInsn->IsStore()) { + auto &defOpnd = static_cast(curInsn.GetOperand(kInsnFirstOpnd)); + for (Insn *cursor = curInsn.GetNext(); cursor != nullptr; cursor = cursor->GetNext()) { + if (!cursor->IsMachineInstruction()) { + continue; + } + uint32 opndNum = cursor->GetOperandSize(); + for (uint32 i = 0; i < opndNum; ++i) { + if (cursor->OpndIsDef(i)) { + continue; + } + if (!cursor->GetOperand(i).IsRegister()) { + continue; + } + auto &useOpnd = static_cast(cursor->GetOperand(i)); + if (useOpnd.GetRegisterNumber() == defOpnd.GetRegisterNumber() && + curInsn.GetOperandSize(kInsnFirstOpnd) != cursor->GetOperandSize(i)) { + return true; + } + } + } } - auto &memOpnd2 = static_cast(insn2.GetOperand(kInsnSecondOpnd)); - if (memOpnd2.GetAddrMode() != MemOperand::kAddrModeBOi || !memOpnd1.IsIntactIndexed()) { + return false; +} + +bool RemoveIdenticalLoadAndStorePattern::CheckCondition(Insn &insn) +{ + if (!cgFunc->GetMirModule().IsCModule()) { return false; } - Operand *base1 = memOpnd1.GetBaseRegister(); - Operand *base2 = memOpnd2.GetBaseRegister(); - if (!((base1 != nullptr) && base1->IsRegister()) || !((base2 != nullptr) && base2->IsRegister())) { + if (!insn.IsStore() && !insn.IsLoad()) { return false; } - - regno_t baseRegNO1 = static_cast(base1)->GetRegisterNumber(); - /* First insn re-write base addr reg1 <- [ reg1 + offset ] */ - if (baseRegNO1 == regNO1) { + auto *memOpnd = static_cast(insn.GetMemOpnd()); + CHECK_FATAL(memOpnd != nullptr, "invalid mem instruction"); + if (memOpnd->GetAddrMode() != MemOperand::kAddrModeBOi || memOpnd->GetBaseRegister() == nullptr || + memOpnd->GetOffsetImmediate() == nullptr) { return false; } - - regno_t baseRegNO2 = static_cast(base2)->GetRegisterNumber(); - if (baseRegNO1 != baseRegNO2) { + prevIdenticalInsn = FindPrevIdenticalMemInsn(insn); + if (prevIdenticalInsn == nullptr) { return false; } - - return memOpnd1.GetOffsetImmediate()->GetOffsetValue() == memOpnd2.GetOffsetImmediate()->GetOffsetValue(); + if (HasImplictSizeUse(insn)) { + return false; + } + if (HasMemReferenceBetweenTwoInsns(insn)) { + return false; + } + return true; } -void RemoveIdenticalLoadAndStoreAArch64::Run(BB &bb, Insn &insn) +void RemoveIdenticalLoadAndStorePattern::Run(BB &bb, Insn &insn) { - Insn *nextInsn = insn.GetNextMachineInsn(); - if (nextInsn == nullptr) { + if (!CheckCondition(insn)) { return; } - MOperator mop1 = insn.GetMachineOpcode(); - MOperator mop2 = nextInsn->GetMachineOpcode(); - if ((mop1 == MOP_wstr && mop2 == MOP_wstr) || (mop1 == MOP_xstr && mop2 == MOP_xstr)) { - if (IsMemOperandsIdentical(insn, *nextInsn)) { - bb.RemoveInsn(insn); - } - } else if ((mop1 == MOP_wstr && mop2 == MOP_wldr) || (mop1 == MOP_xstr && mop2 == MOP_xldr)) { - if (IsMemOperandsIdentical(insn, *nextInsn)) { - bb.RemoveInsn(*nextInsn); - } + if (insn.IsStore()) { + bb.RemoveInsn(*prevIdenticalInsn); + } else { + currInsn = insn.GetNextMachineInsn(); + bb.RemoveInsn(insn); } + optSuccess = true; } bool RemoveIdenticalLoadAndStoreAArch64::IsMemOperandsIdentical(const Insn &insn1, const Insn &insn2) const @@ -2342,6 +3472,25 @@ bool RemoveIdenticalLoadAndStoreAArch64::IsMemOperandsIdentical(const Insn &insn return memOpnd1.GetOffsetImmediate()->GetOffsetValue() == memOpnd2.GetOffsetImmediate()->GetOffsetValue(); } +void RemoveIdenticalLoadAndStoreAArch64::Run(BB &bb, Insn &insn) +{ + Insn *nextInsn = insn.GetNextMachineInsn(); + if (nextInsn == nullptr) { + return; + } + MOperator mop1 = insn.GetMachineOpcode(); + MOperator mop2 = nextInsn->GetMachineOpcode(); + if ((mop1 == MOP_wstr && mop2 == MOP_wstr) || (mop1 == MOP_xstr && mop2 == MOP_xstr)) { + if (IsMemOperandsIdentical(insn, *nextInsn)) { + bb.RemoveInsn(insn); + } + } else if ((mop1 == MOP_wstr && mop2 == MOP_wldr) || (mop1 == MOP_xstr && mop2 == MOP_xldr)) { + if (IsMemOperandsIdentical(insn, *nextInsn)) { + bb.RemoveInsn(*nextInsn); + } + } +} + bool RemoveMovingtoSameRegPattern::CheckCondition(Insn &insn) { DEBUG_ASSERT(insn.GetOperand(kInsnFirstOpnd).IsRegister(), "expects registers"); @@ -2375,6 +3524,63 @@ void RemoveMovingtoSameRegAArch64::Run(BB &bb, Insn &insn) } } +bool MulImmToShiftPattern::CheckCondition(Insn &insn) +{ + auto &useReg = static_cast(insn.GetOperand(kInsnThirdOpnd)); + movInsn = ssaInfo->GetDefInsn(useReg); + if (movInsn == nullptr) { + return false; + } + MOperator prevMop = movInsn->GetMachineOpcode(); + if (prevMop != MOP_wmovri32 && prevMop != MOP_xmovri64) { + return false; + } + ImmOperand &immOpnd = static_cast(movInsn->GetOperand(kInsnSecondOpnd)); + if (immOpnd.IsNegative()) { + return false; + } + uint64 immVal = static_cast(immOpnd.GetValue()); + if (immVal == 0) { + shiftVal = 0; + newMop = insn.GetMachineOpcode() == MOP_xmulrrr ? MOP_xmovri64 : MOP_wmovri32; + return true; + } + /* power of 2 */ + if ((immVal & (immVal - 1)) != 0) { + return false; + } + shiftVal = static_cast(log2(immVal)); + newMop = (prevMop == MOP_xmovri64) ? MOP_xlslrri6 : MOP_wlslrri5; + return true; +} + +void MulImmToShiftPattern::Run(BB &bb, Insn &insn) +{ + /* mov x0,imm and mul to shift */ + if (!CheckCondition(insn)) { + return; + } + auto *aarch64CGFunc = static_cast(cgFunc); + ImmOperand &immOpnd = aarch64CGFunc->CreateImmOperand(shiftVal, k32BitSize, false); + Insn *newInsn; + if (newMop == MOP_xmovri64 || newMop == MOP_wmovri32) { + newInsn = &cgFunc->GetInsnBuilder()->BuildInsn(newMop, insn.GetOperand(kInsnFirstOpnd), immOpnd); + } else { + newInsn = &cgFunc->GetInsnBuilder()->BuildInsn(newMop, insn.GetOperand(kInsnFirstOpnd), + insn.GetOperand(kInsnSecondOpnd), immOpnd); + } + bb.ReplaceInsn(insn, *newInsn); + /* update ssa info */ + ssaInfo->ReplaceInsn(insn, *newInsn); + optSuccess = true; + SetCurrInsn(newInsn); + if (CG_PEEP_DUMP) { + std::vector prevs; + prevs.emplace_back(movInsn); + DumpAfterPattern(prevs, &insn, newInsn); + } +} + bool EnhanceStrLdrAArch64::CheckOperandIsDeadFromInsn(const RegOperand ®Opnd, Insn &insn) { for (uint32 i = 0; i < insn.GetOperandSize(); ++i) { @@ -2396,12 +3602,12 @@ ImmOperand *EnhanceStrLdrAArch64::GetInsnAddOrSubNewOffset(Insn &insn, ImmOperan VaryType vary = offset.GetVary(); auto mOp = insn.GetMachineOpcode(); if (mOp == MOP_xaddrri12 || mOp == MOP_xsubrri12) { - auto &immOpnd = static_cast(insn.GetOperand(kInsnThirdOpnd)); + auto &immOpnd = static_cast(insn.GetOperand(kInsnThirdOpnd)); val = immOpnd.GetValue(); CHECK_FATAL(!(vary == kUnAdjustVary && immOpnd.GetVary() == kUnAdjustVary), "NIY, can not deal this case!"); vary = immOpnd.GetVary(); } else { - auto &immOpnd = static_cast(insn.GetOperand(kInsnThirdOpnd)); + auto &immOpnd = static_cast(insn.GetOperand(kInsnThirdOpnd)); auto &shiftOpnd = static_cast(insn.GetOperand(kInsnFourthOpnd)); CHECK_FATAL(shiftOpnd.GetShiftAmount() == 12, "invalid shiftAmount"); val = (immOpnd.GetValue() << shiftOpnd.GetShiftAmount()); @@ -2411,7 +3617,7 @@ ImmOperand *EnhanceStrLdrAArch64::GetInsnAddOrSubNewOffset(Insn &insn, ImmOperan val = -val; } val += offset.GetValue(); - auto &newImm = static_cast(cgFunc).GetOrCreateOfstOpnd(val, k64BitSize); + auto &newImm = static_cast(cgFunc).GetOrCreateOfstOpnd(val, k64BitSize); newImm.SetVary(vary); return &newImm; } @@ -2420,19 +3626,19 @@ void EnhanceStrLdrAArch64::OptimizeAddrBOI(Insn &insn, MemOperand &memOpnd, Insn { auto *oriBase = memOpnd.GetBaseRegister(); auto *oriOffset = memOpnd.GetOffsetOperand(); - auto &defOpnd = static_cast(prevInsn.GetOperand(kInsnFirstOpnd)); - if (defOpnd.GetRegisterNumber() != oriBase->GetRegisterNumber() || !CheckOperandIsDeadFromInsn(defOpnd, insn)) { + auto &defOpnd = static_cast(prevInsn.GetOperand(kInsnFirstOpnd)); + if (defOpnd.GetRegisterNumber() != oriBase->GetRegisterNumber() || !CheckOperandIsDeadFromInsn(defOpnd, insn)) { return; } - auto *newBase = static_cast(&prevInsn.GetOperand(kInsnSecondOpnd)); + auto *newBase = static_cast(&prevInsn.GetOperand(kInsnSecondOpnd)); auto *newOffset = GetInsnAddOrSubNewOffset(prevInsn, *memOpnd.GetOffsetOperand()); if (newOffset->GetValue() < 0) { - return; // obj dump cannot deal str x19, [x29,#-16] + return; // obj dump cannot deal str x19, [x29,#-16] } memOpnd.SetBaseRegister(*newBase); memOpnd.SetOffsetOperand(*newOffset); - if (!static_cast(cgFunc).IsOperandImmValid(insn.GetMachineOpcode(), &memOpnd, kInsnSecondOpnd)) { + if (!static_cast(cgFunc).IsOperandImmValid(insn.GetMachineOpcode(), &memOpnd, kInsnSecondOpnd)) { // If new offset is invalid, undo it memOpnd.SetBaseRegister(*oriBase); memOpnd.SetOffsetOperand(*oriOffset); @@ -2449,15 +3655,15 @@ void EnhanceStrLdrAArch64::OptimizeAddrBOrXShiftExtend(Insn &insn, MemOperand &m return; } auto *oriIndex = memOpnd.GetIndexRegister(); - auto &defOpnd = static_cast(shiftExtendInsn.GetOperand(kInsnFirstOpnd)); - if (defOpnd.GetRegisterNumber() != oriIndex->GetRegisterNumber() || !CheckOperandIsDeadFromInsn(defOpnd, insn)) { + auto &defOpnd = static_cast(shiftExtendInsn.GetOperand(kInsnFirstOpnd)); + if (defOpnd.GetRegisterNumber() != oriIndex->GetRegisterNumber() || !CheckOperandIsDeadFromInsn(defOpnd, insn)) { return; } - auto &newIndex = static_cast(shiftExtendInsn.GetOperand(kInsnSecondOpnd)); + auto &newIndex = static_cast(shiftExtendInsn.GetOperand(kInsnSecondOpnd)); bool isSigned = (mOp == MOP_xsxtw64); uint32 shift = 0; if (mOp == MOP_xlslrri6) { - shift = static_cast(static_cast(shiftExtendInsn.GetOperand(kInsnThirdOpnd)).GetValue()); + shift = static_cast(static_cast(shiftExtendInsn.GetOperand(kInsnThirdOpnd)).GetValue()); } const uint32 regSize = insn.GetDesc()->GetOpndDes(kInsnFirstOpnd)->GetSize(); // lsl extend insn shift amount can only be 0 or 1(16-bit def opnd) or 2(32-bit def opnd) or @@ -2465,8 +3671,8 @@ void EnhanceStrLdrAArch64::OptimizeAddrBOrXShiftExtend(Insn &insn, MemOperand &m // 32-bit & 64-bit situation now if ((shift == k0BitSize) || (regSize == k32BitSize && shift == k2BitSize) || (regSize == k64BitSize && shift == k3BitSize)) { - auto *newMemOpnd = static_cast(cgFunc).CreateMemOperand(MemOperand::kAddrModeBOrX, - memOpnd.GetSize(), *memOpnd.GetBaseRegister(), newIndex, shift, isSigned); + auto *newMemOpnd = static_cast(cgFunc).CreateMemOperand( + MemOperand::kAddrModeBOrX, memOpnd.GetSize(), *memOpnd.GetBaseRegister(), newIndex, shift, isSigned); insn.SetOperand(kInsnSecondOpnd, *newMemOpnd); shiftExtendInsn.GetBB()->RemoveInsn(shiftExtendInsn); } @@ -2478,12 +3684,12 @@ void EnhanceStrLdrAArch64::OptimizeAddrBOrX(Insn &insn, MemOperand &memOpnd, Ins return; } auto *oriBase = memOpnd.GetBaseRegister(); - auto &defOpnd = static_cast(prevInsn.GetOperand(kInsnFirstOpnd)); + auto &defOpnd = static_cast(prevInsn.GetOperand(kInsnFirstOpnd)); if (defOpnd.GetRegisterNumber() != oriBase->GetRegisterNumber() || !CheckOperandIsDeadFromInsn(defOpnd, insn)) { return; } - auto *newBase = static_cast(&prevInsn.GetOperand(kInsnSecondOpnd)); - auto *newIndex = static_cast(&prevInsn.GetOperand(kInsnThirdOpnd)); + auto *newBase = static_cast(&prevInsn.GetOperand(kInsnSecondOpnd)); + auto *newIndex = static_cast(&prevInsn.GetOperand(kInsnThirdOpnd)); memOpnd.SetBaseRegister(*newBase); memOpnd.SetIndexRegister(*newIndex); @@ -2501,18 +3707,18 @@ void EnhanceStrLdrAArch64::OptimizeWithAddrrrs(Insn &insn, MemOperand &memOpnd, return; } auto *oriBase = memOpnd.GetBaseRegister(); - auto &defOpnd = static_cast(addInsn.GetOperand(kInsnFirstOpnd)); - if (defOpnd.GetRegisterNumber() != oriBase->GetRegisterNumber() || !CheckOperandIsDeadFromInsn(defOpnd, insn)) { + auto &defOpnd = static_cast(addInsn.GetOperand(kInsnFirstOpnd)); + if (defOpnd.GetRegisterNumber() != oriBase->GetRegisterNumber() || !CheckOperandIsDeadFromInsn(defOpnd, insn)) { return; } - auto &newBase = static_cast(addInsn.GetOperand(kInsnSecondOpnd)); - auto &newIndex = static_cast(addInsn.GetOperand(kInsnThirdOpnd)); - auto &shift = static_cast(addInsn.GetOperand(kInsnFourthOpnd)); + auto &newBase = static_cast(addInsn.GetOperand(kInsnSecondOpnd)); + auto &newIndex = static_cast(addInsn.GetOperand(kInsnThirdOpnd)); + auto &shift = static_cast(addInsn.GetOperand(kInsnFourthOpnd)); if (shift.GetShiftOp() != BitShiftOperand::kLSL) { return; } - auto *newMemOpnd = static_cast(cgFunc).CreateMemOperand(MemOperand::kAddrModeBOrX, - memOpnd.GetSize(), newBase, newIndex, shift.GetShiftAmount()); + auto *newMemOpnd = static_cast(cgFunc).CreateMemOperand( + MemOperand::kAddrModeBOrX, memOpnd.GetSize(), newBase, newIndex, shift.GetShiftAmount()); insn.SetOperand(kInsnSecondOpnd, *newMemOpnd); addInsn.GetBB()->RemoveInsn(addInsn); } @@ -2521,7 +3727,7 @@ void EnhanceStrLdrAArch64::Run(BB &bb, Insn &insn) { Operand &opnd = insn.GetOperand(kInsnSecondOpnd); CHECK_FATAL(opnd.IsMemoryAccessOperand(), "Unexpected operand in EnhanceStrLdrAArch64"); - auto &memOpnd = static_cast(opnd); + auto &memOpnd = static_cast(opnd); if (memOpnd.GetAddrMode() != MemOperand::kAddrModeBOi || !memOpnd.GetOffsetImmediate()->IsImmOffset()) { return; } @@ -2529,12 +3735,12 @@ void EnhanceStrLdrAArch64::Run(BB &bb, Insn &insn) auto *prev = insn.GetPreviousMachineInsn(); while (prev != nullptr) { if (prev->GetMachineOpcode() == MOP_xmovrr) { - auto &defOpnd = static_cast(prev->GetOperand(kInsnFirstOpnd)); + auto &defOpnd = static_cast(prev->GetOperand(kInsnFirstOpnd)); if (defOpnd.GetRegisterNumber() != memOpnd.GetBaseRegister()->GetRegisterNumber() || !CheckOperandIsDeadFromInsn(defOpnd, insn)) { return; } - memOpnd.SetBaseRegister(static_cast(prev->GetOperand(kInsnSecondOpnd))); + memOpnd.SetBaseRegister(static_cast(prev->GetOperand(kInsnSecondOpnd))); auto *tmpInsn = prev; prev = prev->GetPreviousMachineInsn(); tmpInsn->GetBB()->RemoveInsn(*tmpInsn); @@ -2561,74 +3767,70 @@ bool IsSameRegisterOperation(const RegOperand &desMovOpnd, const RegOperand &uxt (uxtDestOpnd.GetRegisterNumber() == uxtFromOpnd.GetRegisterNumber())); } -bool CombineContiLoadAndStorePattern::IsRegNotSameMemUseInInsn(const Insn &insn, regno_t regNO, bool isStore, - int64 baseOfst) const +bool CombineContiLoadAndStorePattern::IsRegNotSameMemUseInInsn(const Insn &checkInsn, const Insn &curInsn, + regno_t curBaseRegNO, bool isCurStore, int64 curBaseOfst, + int64 curMemRange) const { - uint32 opndNum = insn.GetOperandSize(); - bool sameMemAccess = false; /* both store or load */ - if (insn.IsStore() == isStore) { - sameMemAccess = true; - } + uint32 opndNum = checkInsn.GetOperandSize(); for (uint32 i = 0; i < opndNum; ++i) { - Operand &opnd = insn.GetOperand(i); + Operand &opnd = checkInsn.GetOperand(i); if (opnd.IsList()) { - auto &listOpnd = static_cast(opnd); - for (auto listElem : listOpnd.GetOperands()) { - RegOperand *regOpnd = static_cast(listElem); + auto &listOpnd = static_cast(opnd); + for (auto &listElem : listOpnd.GetOperands()) { + auto *regOpnd = static_cast(listElem); DEBUG_ASSERT(regOpnd != nullptr, "parameter operand must be RegOperand"); - if (regNO == regOpnd->GetRegisterNumber()) { + if (curBaseRegNO == regOpnd->GetRegisterNumber()) { return true; } } } else if (opnd.IsMemoryAccessOperand()) { auto &memOperand = static_cast(opnd); - RegOperand *base = memOperand.GetBaseRegister(); - /* need check offset as well */ + RegOperand *checkBaseReg = memOperand.GetBaseRegister(); + // If the BASEREG of the two MEM insns are different, we use cg-mem-reference to check the alias of two MEM: + // if there is no alias, we can combine MEM pair cross the MEM insn. + // e.g. + // str x1, [x9] + // str x6, [x2] + // str x3, [x9, #8] regno_t stackBaseRegNO = cgFunc->UseFP() ? R29 : RSP; - if (!sameMemAccess && base != nullptr) { - regno_t curBaseRegNO = base->GetRegisterNumber(); - int64 memBarrierRange = static_cast(insn.IsLoadStorePair() ? k16BitSize : k8BitSize); - if (!(curBaseRegNO == regNO && memOperand.GetAddrMode() == MemOperand::kAddrModeBOi && - memOperand.GetOffsetImmediate() != nullptr && - (memOperand.GetOffsetImmediate()->GetOffsetValue() <= (baseOfst - memBarrierRange) || - memOperand.GetOffsetImmediate()->GetOffsetValue() >= (baseOfst + memBarrierRange)))) { - return true; - } - } - /* do not trust the following situation : - * str x1, [x9] - * str x6, [x2] - * str x3, [x9, #8] - */ - if (isStore && regNO != stackBaseRegNO && base != nullptr && base->GetRegisterNumber() != stackBaseRegNO && - base->GetRegisterNumber() != regNO) { + if ((isCurStore || checkInsn.IsStore()) && checkBaseReg != nullptr && + !(curBaseRegNO == stackBaseRegNO && checkBaseReg->GetRegisterNumber() == stackBaseRegNO) && + checkBaseReg->GetRegisterNumber() != curBaseRegNO && + AArch64MemReference::HasAliasMemoryDep(checkInsn, curInsn, kDependenceTypeNone)) { return true; } - if (isStore && base != nullptr && base->GetRegisterNumber() == regNO) { - if (memOperand.GetAddrMode() == MemOperand::kAddrModeBOi && - memOperand.GetOffsetImmediate() != nullptr) { - int64 curOffset = memOperand.GetOffsetImmediate()->GetOffsetValue(); - if (memOperand.GetSize() == k64BitSize) { - uint32 memBarrierRange = insn.IsLoadStorePair() ? k16BitSize : k8BitSize; - if (curOffset < baseOfst + memBarrierRange && curOffset > baseOfst - memBarrierRange) { - return true; - } - } else if (memOperand.GetSize() == k32BitSize) { - uint32 memBarrierRange = insn.IsLoadStorePair() ? k8BitSize : k4BitSize; - if (curOffset < baseOfst + memBarrierRange && curOffset > baseOfst - memBarrierRange) { - return true; - } + // Check memory overlap + if ((isCurStore || checkInsn.IsStore()) && checkBaseReg != nullptr && + memOperand.GetAddrMode() == MemOperand::kAddrModeBOi && memOperand.GetOffsetImmediate() != nullptr) { + // If memInsn is split with x16, we need to find the actual base register + int64 checkOffset = memOperand.GetOffsetImmediate()->GetOffsetValue(); + regno_t checkRegNO = checkBaseReg->GetRegisterNumber(); + if (checkRegNO == R16) { + const Insn *prevInsn = checkInsn.GetPrev(); + // Before cgaggressiveopt, the def and use of R16 must be adjacent, and the def of R16 must be + // addrri, otherwise, the process is conservative and the mem insn that can be combined is not + // search forward. + if (prevInsn == nullptr || prevInsn->GetMachineOpcode() != MOP_xaddrri12 || + static_cast(prevInsn->GetOperand(kInsnFirstOpnd)).GetRegisterNumber() != R16) { + return true; } + checkOffset += static_cast(prevInsn->GetOperand(kInsnThirdOpnd)).GetValue(); + } + auto checkMemRange = static_cast(checkInsn.GetMemoryByteSize()); + // curOfst curOfst+curMemRange + // |______|_/_/_/_/_/_/_/_/_/_/_|____________| + if ((curBaseOfst >= checkOffset && curBaseOfst < (checkOffset + checkMemRange)) || + (checkOffset >= curBaseOfst && checkOffset < (curBaseOfst + curMemRange))) { + return true; } } } else if (opnd.IsConditionCode()) { - Operand &rflagOpnd = cgFunc->GetOrCreateRflag(); - RegOperand &rflagReg = static_cast(rflagOpnd); - if (rflagReg.GetRegisterNumber() == regNO) { + auto &rflagOpnd = static_cast(cgFunc->GetOrCreateRflag()); + if (rflagOpnd.GetRegisterNumber() == curBaseRegNO) { return true; } } else if (opnd.IsRegister()) { - if (!isStore && static_cast(opnd).GetRegisterNumber() == regNO) { + if (!isCurStore && static_cast(opnd).GetRegisterNumber() == curBaseRegNO) { return true; } } @@ -2637,10 +3839,9 @@ bool CombineContiLoadAndStorePattern::IsRegNotSameMemUseInInsn(const Insn &insn, } std::vector CombineContiLoadAndStorePattern::FindPrevStrLdr(Insn &insn, regno_t destRegNO, regno_t memBaseRegNO, - int64 baseOfst) + int64 baseOfst) const { std::vector prevContiInsns; - bool isStr = insn.IsStore(); for (Insn *curInsn = insn.GetPrev(); curInsn != nullptr; curInsn = curInsn->GetPrev()) { if (!curInsn->IsMachineInstruction()) { continue; @@ -2648,39 +3849,52 @@ std::vector CombineContiLoadAndStorePattern::FindPrevStrLdr(Insn &insn, if (curInsn->IsRegDefined(memBaseRegNO)) { return prevContiInsns; } - if (IsRegNotSameMemUseInInsn(*curInsn, memBaseRegNO, insn.IsStore(), static_cast(baseOfst))) { + DEBUG_ASSERT(insn.GetOperand(kInsnSecondOpnd).IsMemoryAccessOperand(), "invalid mem insn"); + auto baseMemRange = static_cast(insn.GetMemoryByteSize()); + if (IsRegNotSameMemUseInInsn(*curInsn, insn, memBaseRegNO, insn.IsStore(), static_cast(baseOfst), + baseMemRange)) { return prevContiInsns; } - /* return continuous STD/LDR insn */ - if (((isStr && curInsn->IsStore()) || (!isStr && curInsn->IsLoad())) && !curInsn->IsLoadStorePair()) { + // record continuous STD/LDR insn + if (!curInsn->IsLoadStorePair() && + ((insn.IsStore() && curInsn->IsStore()) || (insn.IsLoad() && curInsn->IsLoad()))) { auto *memOperand = static_cast(curInsn->GetMemOpnd()); /* do not combine ldr r0, label */ if (memOperand != nullptr) { - auto *BaseRegOpnd = static_cast(memOperand->GetBaseRegister()); - DEBUG_ASSERT(BaseRegOpnd == nullptr || !BaseRegOpnd->IsVirtualRegister(), + auto *baseRegOpnd = static_cast(memOperand->GetBaseRegister()); + DEBUG_ASSERT(baseRegOpnd == nullptr || !baseRegOpnd->IsVirtualRegister(), "physical register has not been allocated?"); if (memOperand->GetAddrMode() == MemOperand::kAddrModeBOi && - BaseRegOpnd->GetRegisterNumber() == memBaseRegNO) { + baseRegOpnd->GetRegisterNumber() == memBaseRegNO) { prevContiInsns.emplace_back(curInsn); } } } /* check insn that changes the data flow */ - regno_t stackBaseRegNO = cgFunc->UseFP() ? R29 : RSP; /* ldr x8, [x21, #8] * call foo() * ldr x9, [x21, #16] * although x21 is a calleeSave register, there is no guarantee data in memory [x21] is not changed */ - if (curInsn->IsCall() && - (!AArch64Abi::IsCalleeSavedReg(static_cast(destRegNO)) || memBaseRegNO != stackBaseRegNO)) { + if (curInsn->IsCall() || curInsn->GetMachineOpcode() == MOP_asm) { return prevContiInsns; } + /* Check regOpnd for mem access: + * 1. if the destRegNO is RZR, we do not need to check define and use for destRegNO between PREVINSN and INSN; + * 2. for load insn, we forbid both use and define destRegNO between PREVINSN and INSN; + * 3. for store insn, we only forbit define destRegNO between PREVINSN and INSN; + * e.g.1 + * ldr x2, [sp, #16] + * add x3, x1, #5 & add x1, x3, #5 ---\--> all [x1] use and define can not across + * ldr x1, [sp, #8] + * e.g.2 + * str x2, [sp, #16] + * add x1, x3, #5 ---\---> only [x1] define can not across + * str x1, [sp, #8] + */ /* store opt should not cross call due to stack args */ - if (curInsn->IsCall() && isStr) { - return prevContiInsns; - } - if (curInsn->GetMachineOpcode() == MOP_asm) { + if (destRegNO != RZR && + ((insn.IsLoad() && curInsn->ScanReg(destRegNO)) || (insn.IsStore() && curInsn->IsRegDefined(destRegNO)))) { return prevContiInsns; } if (curInsn->ScanReg(destRegNO)) { @@ -2690,297 +3904,211 @@ std::vector CombineContiLoadAndStorePattern::FindPrevStrLdr(Insn &insn, return prevContiInsns; } -Insn *CombineContiLoadAndStorePattern::FindValidSplitAddInsn(Insn &curInsn, RegOperand &baseOpnd) const +bool CombineContiLoadAndStorePattern::CheckCondition(Insn &insn) { - Insn *splitAdd = nullptr; - for (Insn *cursor = curInsn.GetPrev(); cursor != nullptr; cursor = cursor->GetPrev()) { - if (!cursor->IsMachineInstruction()) { - continue; - } - if (cursor->IsCall()) { - break; - } - if (cursor->IsRegDefined(baseOpnd.GetRegisterNumber())) { - break; - } - MOperator mOp = cursor->GetMachineOpcode(); - if (mOp != MOP_xaddrri12 && mOp != MOP_waddrri12) { - continue; - } - auto &destOpnd = static_cast(cursor->GetOperand(kInsnFirstOpnd)); - if (destOpnd.GetRegisterNumber() != R16 || destOpnd.GetSize() != baseOpnd.GetSize()) { + MOperator mop = insn.GetMachineOpcode(); + if (mop == MOP_wldrb || mop == MOP_wldrh) { + return false; + } + auto *curMemOpnd = static_cast(insn.GetMemOpnd()); + DEBUG_ASSERT(curMemOpnd != nullptr, "get mem operand failed"); + if (!doAggressiveCombine || curMemOpnd->GetAddrMode() != MemOperand::kAddrModeBOi) { + return false; + } + return true; +} + +/* Combining 2 STRs into 1 stp or 2 LDRs into 1 ldp */ +void CombineContiLoadAndStorePattern::Run(BB &bb, Insn &insn) +{ + if (!CheckCondition(insn)) { + return; + } + + auto *curMemOpnd = static_cast(insn.GetMemOpnd()); + DEBUG_ASSERT(curMemOpnd->GetAddrMode() == MemOperand::kAddrModeBOi, "invalid continues mem insn"); + OfstOperand *curOfstOpnd = curMemOpnd->GetOffsetImmediate(); + int64 curOfstVal = curOfstOpnd ? curOfstOpnd->GetOffsetValue() : 0; + + auto *baseRegOpnd = static_cast(curMemOpnd->GetBaseRegister()); + DEBUG_ASSERT(baseRegOpnd == nullptr || !baseRegOpnd->IsVirtualRegister(), + "physical register has not been allocated?"); + auto &curDestOpnd = static_cast(insn.GetOperand(kInsnFirstOpnd)); + std::vector prevContiInsnVec = + FindPrevStrLdr(insn, curDestOpnd.GetRegisterNumber(), baseRegOpnd->GetRegisterNumber(), curOfstVal); + for (auto prevContiInsn : prevContiInsnVec) { + DEBUG_ASSERT(prevContiInsn != nullptr, "get previous consecutive instructions failed"); + auto *prevMemOpnd = static_cast(prevContiInsn->GetMemOpnd()); + DEBUG_ASSERT(prevMemOpnd->GetAddrMode() == MemOperand::kAddrModeBOi, "invalid continues mem insn"); + OfstOperand *prevOfstOpnd = prevMemOpnd->GetOffsetImmediate(); + int64 prevOfstVal = prevOfstOpnd ? prevOfstOpnd->GetOffsetValue() : 0; + auto &prevDestOpnd = static_cast(prevContiInsn->GetOperand(kInsnFirstOpnd)); + if (prevDestOpnd.GetRegisterType() != curDestOpnd.GetRegisterType()) { continue; } - auto &useOpnd = static_cast(cursor->GetOperand(kInsnSecondOpnd)); - /* - * split add as following: - * add R16, R0, #2, LSL #12 - * add R16, R16, #1536 - */ - if (useOpnd.GetRegisterNumber() != baseOpnd.GetRegisterNumber()) { - if (useOpnd.GetRegisterNumber() == R16) { - Insn *defInsn = cursor->GetPrev(); - CHECK_FATAL(defInsn, "invalid defInsn"); - CHECK_FATAL(defInsn->GetMachineOpcode() == MOP_xaddrri24 || - defInsn->GetMachineOpcode() == MOP_waddrri24, - "split with wrong add"); - auto &opnd = static_cast(defInsn->GetOperand(kInsnSecondOpnd)); - if (opnd.GetRegisterNumber() == baseOpnd.GetRegisterNumber()) { - splitAdd = cursor; + + MemOperand *combineMemOpnd = (curOfstVal < prevOfstVal) ? curMemOpnd : prevMemOpnd; + if (IsValidNormalLoadOrStorePattern(insn, *prevContiInsn, *curMemOpnd, curOfstVal, prevOfstVal)) { + // Process normal mem pair + MOperator newMop = GetMopPair(insn.GetMachineOpcode(), true); + Insn *combineInsn = + GenerateMemPairInsn(newMop, curDestOpnd, prevDestOpnd, *combineMemOpnd, curOfstVal < prevOfstVal); + DEBUG_ASSERT(combineInsn != nullptr, "create combineInsn failed"); + bb.InsertInsnAfter(*prevContiInsn, *combineInsn); + if (!(static_cast(*cgFunc).IsOperandImmValid( + newMop, combineMemOpnd, isPairAfterCombine ? kInsnThirdOpnd : kInsnSecondOpnd))) { + if (FindUseX16AfterInsn(*prevContiInsn)) { + // Do not combine Insns when x16 was used after curInsn + bb.RemoveInsn(*combineInsn); + return; } + SPLIT_INSN(combineInsn, cgFunc); } - break; - } else { - splitAdd = cursor; - break; + RemoveInsnAndKeepComment(bb, insn, *prevContiInsn); + SetCurrInsn(combineInsn); + optSuccess = true; + return; + } else if (IsValidStackArgLoadOrStorePattern(insn, *prevContiInsn, *curMemOpnd, *prevMemOpnd, curOfstVal, + prevOfstVal)) { + // Process stack-arg mem pair + regno_t curDestRegNo = curDestOpnd.GetRegisterNumber(); + regno_t prevDestRegNo = prevDestOpnd.GetRegisterNumber(); + RegOperand &newDest = static_cast(cgFunc)->GetOrCreatePhysicalRegisterOperand( + static_cast(curDestRegNo), k64BitSize, curDestOpnd.GetRegisterType()); + RegOperand &newPrevDest = static_cast(cgFunc)->GetOrCreatePhysicalRegisterOperand( + static_cast(prevDestRegNo), k64BitSize, prevDestOpnd.GetRegisterType()); + MOperator newMop = (curDestOpnd.GetRegisterType() == kRegTyInt) ? MOP_xstp : MOP_dstp; + if (!(static_cast(*cgFunc).IsOperandImmValid(newMop, combineMemOpnd, kInsnThirdOpnd))) { + return; + } + Insn *combineInsn = + GenerateMemPairInsn(newMop, newDest, newPrevDest, *combineMemOpnd, curOfstVal < prevOfstVal); + bb.InsertInsnAfter(*prevContiInsn, *combineInsn); + RemoveInsnAndKeepComment(bb, insn, *prevContiInsn); + SetCurrInsn(combineInsn); + optSuccess = true; + return; } } - return splitAdd; } -bool CombineContiLoadAndStorePattern::PlaceSplitAddInsn(const Insn &curInsn, Insn &combineInsn, - const MemOperand &memOperand, RegOperand &baseOpnd, - uint32 bitLen) const +bool CombineContiLoadAndStorePattern::FindUseX16AfterInsn(const Insn &curInsn) const { - Insn *cursor = nullptr; - MemOperand *maxOfstMem = nullptr; - int64 maxOfstVal = 0; - MOperator mop = curInsn.GetMachineOpcode(); - OfstOperand *ofstOpnd = memOperand.GetOffsetImmediate(); - int64 ofstVal = ofstOpnd->GetOffsetValue(); - auto &aarFunc = static_cast(*cgFunc); - for (cursor = curInsn.GetNext(); cursor != nullptr; cursor = cursor->GetNext()) { + for (Insn *cursor = curInsn.GetNext(); cursor != nullptr; cursor = cursor->GetNext()) { if (!cursor->IsMachineInstruction()) { continue; } - if (cursor->GetMachineOpcode() == mop && (cursor->IsLoad() || cursor->IsStore())) { - auto &curMemOpnd = static_cast(cursor->GetOperand(kInsnSecondOpnd)); - RegOperand *curBaseOpnd = curMemOpnd.GetBaseRegister(); - if (curMemOpnd.GetAddrMode() == MemOperand::kAddrModeBOi && RegOperand::IsSameReg(baseOpnd, *curBaseOpnd)) { - OfstOperand *curOfstOpnd = curMemOpnd.GetOffsetImmediate(); - CHECK_FATAL(curOfstOpnd, "invalid OfstOperand"); - if (curOfstOpnd->GetOffsetValue() > ofstVal && - (curOfstOpnd->GetOffsetValue() - ofstVal) < MemOperand::GetMaxPairPIMM(bitLen) && - !aarFunc.IsOperandImmValid(combineInsn.GetMachineOpcode(), &curMemOpnd, kInsnThirdOpnd)) { - maxOfstMem = &curMemOpnd; - maxOfstVal = curOfstOpnd->GetOffsetValue(); - } + for (uint32 defRegNo : cursor->GetDefRegs()) { + if (defRegNo == R16) { + return false; } } - if (cursor->IsRegDefined(baseOpnd.GetRegisterNumber())) { - break; + if ((!cursor->IsLoad() && !cursor->IsStore() && !cursor->IsLoadStorePair()) || cursor->IsAtomic()) { + continue; } - if (cursor->IsRegDefined(R16)) { - break; + const InsnDesc *md = &AArch64CG::kMd[cursor->GetMachineOpcode()]; + if (cursor->IsLoadLabel() || md->IsLoadAddress()) { + continue; } - } - MemOperand *newMemOpnd = nullptr; - if (maxOfstMem == nullptr) { - newMemOpnd = &aarFunc.SplitOffsetWithAddInstruction(memOperand, bitLen, static_cast(R16), false, - &combineInsn, true); - } else { - RegOperand *addResOpnd = aarFunc.GetBaseRegForSplit(R16); - ImmOperand &immAddend = - aarFunc.SplitAndGetRemained(*maxOfstMem, bitLen, addResOpnd, maxOfstVal, false, &combineInsn, true); - newMemOpnd = &aarFunc.CreateReplacementMemOperand(bitLen, *addResOpnd, ofstVal - immAddend.GetValue()); - if (!(aarFunc.IsOperandImmValid(combineInsn.GetMachineOpcode(), newMemOpnd, kInsnThirdOpnd))) { - newMemOpnd = &aarFunc.SplitOffsetWithAddInstruction(memOperand, bitLen, static_cast(R16), false, - &combineInsn, true); - } else { - aarFunc.SelectAddAfterInsn(*addResOpnd, baseOpnd, immAddend, PTY_i64, false, combineInsn); + uint32 memIdx = (cursor->IsLoadStorePair() ? kInsnThirdOpnd : kInsnSecondOpnd); + auto &curMemOpnd = static_cast(cursor->GetOperand(memIdx)); + RegOperand *baseOpnd = curMemOpnd.GetBaseRegister(); + if (baseOpnd != nullptr && baseOpnd->GetRegisterNumber() == R16) { + return true; } } - if (!(aarFunc.IsOperandImmValid(combineInsn.GetMachineOpcode(), newMemOpnd, kInsnThirdOpnd))) { - return false; - } - combineInsn.SetOperand(kInsnThirdOpnd, *newMemOpnd); - return true; + return false; } -bool CombineContiLoadAndStorePattern::SplitOfstWithAddToCombine(const Insn &curInsn, Insn &combineInsn, - const MemOperand &memOperand) const +Insn *CombineContiLoadAndStorePattern::GenerateMemPairInsn(MOperator newMop, RegOperand &curDestOpnd, + RegOperand &prevDestOpnd, MemOperand &combineMemOpnd, + bool isCurDestFirst) { - auto *baseRegOpnd = static_cast(memOperand.GetBaseRegister()); - auto *ofstOpnd = static_cast(memOperand.GetOffsetImmediate()); - DEBUG_ASSERT(baseRegOpnd && ofstOpnd, "get baseOpnd and ofstOpnd failed"); - CHECK_FATAL(combineInsn.GetOperand(kInsnFirstOpnd).GetSize() == combineInsn.GetOperand(kInsnSecondOpnd).GetSize(), - "the size must equal"); - if (baseRegOpnd->GetRegisterNumber() == R16) { - return false; - } - Insn *splitAdd = FindValidSplitAddInsn(combineInsn, *baseRegOpnd); - const InsnDesc *md = &AArch64CG::kMd[combineInsn.GetMachineOpcode()]; - auto *opndProp = md->opndMD[kInsnFirstOpnd]; - auto &aarFunc = static_cast(*cgFunc); - if (splitAdd == nullptr) { - if (combineInsn.IsLoadStorePair()) { - if (ofstOpnd->GetOffsetValue() < 0) { - return false; /* do not split */ - } - } - /* create and place addInsn */ - return PlaceSplitAddInsn(curInsn, combineInsn, memOperand, *baseRegOpnd, opndProp->GetSize()); - } else { - auto &newBaseReg = static_cast(splitAdd->GetOperand(kInsnFirstOpnd)); - auto &addImmOpnd = static_cast(splitAdd->GetOperand(kInsnThirdOpnd)); - int64 addVal = 0; - if (static_cast(splitAdd->GetOperand(kInsnSecondOpnd)).GetRegisterNumber() == R16) { - Insn *defInsn = splitAdd->GetPrev(); - CHECK_FATAL(defInsn->GetMachineOpcode() == MOP_xaddrri24 || defInsn->GetMachineOpcode() == MOP_waddrri24, - "split with wrong add"); - auto &immOpnd = static_cast(defInsn->GetOperand(kInsnThirdOpnd)); - auto &shiftOpnd = static_cast(defInsn->GetOperand(kInsnFourthOpnd)); - addVal = (immOpnd.GetValue() << shiftOpnd.GetShiftAmount()) + addImmOpnd.GetValue(); - } else { - addVal = addImmOpnd.GetValue(); - } - auto *newOfstOpnd = - &aarFunc.CreateOfstOpnd(static_cast(ofstOpnd->GetOffsetValue() - addVal), ofstOpnd->GetSize()); - auto *newMemOpnd = aarFunc.CreateMemOperand(MemOperand::kAddrModeBOi, opndProp->GetSize(), newBaseReg, nullptr, - newOfstOpnd, memOperand.GetSymbol()); - if (!(static_cast(*cgFunc).IsOperandImmValid(combineInsn.GetMachineOpcode(), newMemOpnd, - kInsnThirdOpnd))) { - return PlaceSplitAddInsn(curInsn, combineInsn, memOperand, *baseRegOpnd, opndProp->GetSize()); - } - combineInsn.SetOperand(kInsnThirdOpnd, *newMemOpnd); - return true; + DEBUG_ASSERT(newMop != MOP_undef, "invalid MOperator"); + Insn *combineInsn = nullptr; + if (isPairAfterCombine) { // for ldr/str --> ldp/stp + combineInsn = (isCurDestFirst) + ? &cgFunc->GetInsnBuilder()->BuildInsn(newMop, curDestOpnd, prevDestOpnd, combineMemOpnd) + : &cgFunc->GetInsnBuilder()->BuildInsn(newMop, prevDestOpnd, curDestOpnd, combineMemOpnd); + } else { // for strb/strh --> strh/str, curDestOpnd == preDestOpnd + combineInsn = &cgFunc->GetInsnBuilder()->BuildInsn(newMop, curDestOpnd, combineMemOpnd); + combineMemOpnd.SetSize(newMop == MOP_wstrh ? maplebe::k16BitSize : maplebe::k32BitSize); } + return combineInsn; } -bool CombineContiLoadAndStorePattern::CheckCondition(Insn &insn) +bool CombineContiLoadAndStorePattern::IsValidNormalLoadOrStorePattern(const Insn &insn, const Insn &prevInsn, + const MemOperand &memOpnd, int64 curOfstVal, + int64 prevOfstVal) { - memOpnd = static_cast(insn.GetMemOpnd()); - DEBUG_ASSERT(memOpnd != nullptr, "get mem operand failed"); - if (memOpnd->GetAddrMode() != MemOperand::kAddrModeBOi) { + if (memOpnd.IsStackArgMem()) { return false; } - if (!doAggressiveCombine) { + DEBUG_ASSERT(insn.GetOperand(kInsnFirstOpnd).IsRegister(), "unexpect operand"); + DEBUG_ASSERT(prevInsn.GetOperand(kInsnFirstOpnd).IsRegister(), "unexpect operand"); + auto &curDestOpnd = static_cast(insn.GetOperand(kInsnFirstOpnd)); + auto &prevDestOpnd = static_cast(prevInsn.GetOperand(kInsnFirstOpnd)); + if (prevDestOpnd.GetRegisterType() != curDestOpnd.GetRegisterType() || + curDestOpnd.GetSize() != prevDestOpnd.GetSize()) { return false; } - return true; -} - -/* Combining 2 STRs into 1 stp or 2 LDRs into 1 ldp */ -void CombineContiLoadAndStorePattern::Run(BB &bb, Insn &insn) -{ - if (!CheckCondition(insn)) { - return; + uint32 memSize = insn.GetMemoryByteSize(); + uint32 prevMemSize = prevInsn.GetMemoryByteSize(); + if (memSize != prevMemSize) { + return false; } - MOperator thisMop = insn.GetMachineOpcode(); - DEBUG_ASSERT(insn.GetOperand(kInsnFirstOpnd).IsRegister(), "unexpect operand"); - auto &destOpnd = static_cast(insn.GetOperand(kInsnFirstOpnd)); - auto *baseRegOpnd = static_cast(memOpnd->GetBaseRegister()); - OfstOperand *offsetOpnd = memOpnd->GetOffsetImmediate(); - CHECK_FATAL(offsetOpnd != nullptr, "offset opnd lost"); - DEBUG_ASSERT(baseRegOpnd == nullptr || !baseRegOpnd->IsVirtualRegister(), - "physical register has not been allocated?"); - std::vector prevContiInsnVec = FindPrevStrLdr( - insn, destOpnd.GetRegisterNumber(), baseRegOpnd->GetRegisterNumber(), offsetOpnd->GetOffsetValue()); - for (auto prevContiInsn : prevContiInsnVec) { - DEBUG_ASSERT(prevContiInsn != nullptr, "get previous consecutive instructions failed"); - auto *prevMemOpnd = static_cast(prevContiInsn->GetMemOpnd()); - if (memOpnd->GetIndexOpt() != prevMemOpnd->GetIndexOpt()) { - continue; - } - OfstOperand *prevOffsetOpnd = prevMemOpnd->GetOffsetImmediate(); - CHECK_FATAL(offsetOpnd != nullptr && prevOffsetOpnd != nullptr, "both conti str/ldr have no offset"); - auto &prevDestOpnd = static_cast(prevContiInsn->GetOperand(kInsnFirstOpnd)); - uint32 memSize = insn.GetMemoryByteSize(); - uint32 prevMemSize = prevContiInsn->GetMemoryByteSize(); - if (prevDestOpnd.GetRegisterType() != destOpnd.GetRegisterType()) { - continue; + + int64 diffVal = std::abs(curOfstVal - prevOfstVal); + if ((memSize == k1ByteSize && diffVal == k1BitSize) || (memSize == k2ByteSize && diffVal == k2BitSize) || + (memSize == k4ByteSize && diffVal == k4BitSize) || (memSize == k8ByteSize && diffVal == k8BitSize)) { + MOperator curMop = insn.GetMachineOpcode(); + DEBUG_ASSERT(curMop != MOP_wldrb && curMop != MOP_wldrh, "invalid mem insn that cannot be combined"); + if (curMop == MOP_wstrb || curMop == MOP_wstrh) { + isPairAfterCombine = false; } - int64 offsetVal = offsetOpnd->GetOffsetValue(); - int64 prevOffsetVal = prevOffsetOpnd->GetOffsetValue(); - auto diffVal = std::abs(offsetVal - prevOffsetVal); - regno_t destRegNO = destOpnd.GetRegisterNumber(); + + regno_t destRegNO = curDestOpnd.GetRegisterNumber(); regno_t prevDestRegNO = prevDestOpnd.GetRegisterNumber(); - if (insn.IsStore() && memOpnd->IsStackArgMem() && prevMemOpnd->IsStackArgMem() && - (memSize == k4ByteSize || memSize == k8ByteSize) && diffVal == k8BitSize && - (prevMemSize == k4ByteSize || prevMemSize == k8ByteSize) && - (destOpnd.GetValidBitsNum() == memSize * k8BitSize) && - (prevDestOpnd.GetValidBitsNum() == prevMemSize * k8BitSize)) { - RegOperand &newDest = static_cast(cgFunc)->GetOrCreatePhysicalRegisterOperand( - static_cast(destRegNO), k64BitSize, destOpnd.GetRegisterType()); - RegOperand &newPrevDest = static_cast(cgFunc)->GetOrCreatePhysicalRegisterOperand( - static_cast(prevDestRegNO), k64BitSize, prevDestOpnd.GetRegisterType()); - MemOperand *combineMemOpnd = (offsetVal < prevOffsetVal) ? memOpnd : prevMemOpnd; - MOperator mopPair = (destOpnd.GetRegisterType() == kRegTyInt) ? MOP_xstp : MOP_dstp; - if ((static_cast(*cgFunc).IsOperandImmValid(mopPair, combineMemOpnd, kInsnThirdOpnd))) { - Insn &combineInsn = - (offsetVal < prevOffsetVal) - ? cgFunc->GetInsnBuilder()->BuildInsn(mopPair, newDest, newPrevDest, *combineMemOpnd) - : cgFunc->GetInsnBuilder()->BuildInsn(mopPair, newPrevDest, newDest, *combineMemOpnd); - bb.InsertInsnAfter(*prevContiInsn, combineInsn); - RemoveInsnAndKeepComment(bb, insn, *prevContiInsn); - return; - } - } - if (memSize != prevMemSize || thisMop != prevContiInsn->GetMachineOpcode() || - prevDestOpnd.GetSize() != destOpnd.GetSize()) { - continue; + if (destRegNO == RZR && prevDestRegNO == RZR) { + return true; } - /* do combination str/ldr -> stp/ldp */ - if ((insn.IsStore() || destRegNO != prevDestRegNO) || (destRegNO == RZR && prevDestRegNO == RZR)) { - if ((memSize == k8ByteSize && diffVal == k8BitSize) || (memSize == k4ByteSize && diffVal == k4BitSize) || - (memSize == k16ByteSize && diffVal == k16BitSize)) { - MOperator mopPair = GetMopPair(thisMop); - MemOperand *combineMemOpnd = (offsetVal < prevOffsetVal) ? memOpnd : prevMemOpnd; - Insn &combineInsn = - (offsetVal < prevOffsetVal) - ? cgFunc->GetInsnBuilder()->BuildInsn(mopPair, destOpnd, prevDestOpnd, *combineMemOpnd) - : cgFunc->GetInsnBuilder()->BuildInsn(mopPair, prevDestOpnd, destOpnd, *combineMemOpnd); - bb.InsertInsnAfter(*prevContiInsn, combineInsn); - if (!(static_cast(*cgFunc).IsOperandImmValid(mopPair, combineMemOpnd, - kInsnThirdOpnd)) && - !SplitOfstWithAddToCombine(insn, combineInsn, *combineMemOpnd)) { - bb.RemoveInsn(combineInsn); - return; - } - RemoveInsnAndKeepComment(bb, insn, *prevContiInsn); - return; - } + + if (insn.IsLoad() && destRegNO == prevDestRegNO) { + return false; } - /* do combination strb/ldrb -> strh/ldrh -> str/ldr */ - if (destRegNO == prevDestRegNO && destRegNO == RZR && prevDestRegNO == RZR) { - if ((memSize == k1ByteSize && diffVal == k1ByteSize) || (memSize == k2ByteSize && diffVal == k2ByteSize)) { - MOperator mopPair = GetMopHigherByte(thisMop); - if (offsetVal < prevOffsetVal) { - if (static_cast(*cgFunc).IsOperandImmValid(mopPair, memOpnd, kInsnSecondOpnd)) { - Insn &combineInsn = cgFunc->GetInsnBuilder()->BuildInsn(mopPair, destOpnd, *memOpnd); - bb.InsertInsnAfter(*prevContiInsn, combineInsn); - RemoveInsnAndKeepComment(bb, insn, *prevContiInsn); - return; - } - } else { - if (static_cast(*cgFunc).IsOperandImmValid(mopPair, prevMemOpnd, - kInsnSecondOpnd)) { - Insn &combineInsn = cgFunc->GetInsnBuilder()->BuildInsn(mopPair, prevDestOpnd, *prevMemOpnd); - bb.InsertInsnAfter(*prevContiInsn, combineInsn); - RemoveInsnAndKeepComment(bb, insn, *prevContiInsn); - return; - } - } - } + + if ((curMop == MOP_wstrb || curMop == MOP_wstrh) && (destRegNO != RZR || prevDestRegNO != RZR)) { + return false; } + + return true; } + + return false; } -MOperator CombineContiLoadAndStorePattern::GetMopHigherByte(MOperator mop) const +bool CombineContiLoadAndStorePattern::IsValidStackArgLoadOrStorePattern(const Insn &curInsn, const Insn &prevInsn, + const MemOperand &curMemOpnd, + const MemOperand &prevMemOpnd, int64 curOfstVal, + int64 prevOfstVal) const { - switch (mop) { - case MOP_wldrb: - return MOP_wldrh; - case MOP_wstrb: - return MOP_wstrh; - case MOP_wldrh: - return MOP_wldr; - case MOP_wstrh: - return MOP_wstr; - default: - DEBUG_ASSERT(false, "should not run here"); - return MOP_undef; + if (!curInsn.IsStore()) { + return false; + } + if (!curMemOpnd.IsStackArgMem() || !prevMemOpnd.IsStackArgMem()) { + return false; + } + auto &curDestOpnd = static_cast(curInsn.GetOperand(kInsnFirstOpnd)); + auto &prevDestOpnd = static_cast(prevInsn.GetOperand(kInsnFirstOpnd)); + uint32 memSize = curInsn.GetMemoryByteSize(); + uint32 prevMemSize = prevInsn.GetMemoryByteSize(); + auto diffVal = std::abs(curOfstVal - prevOfstVal); + if ((memSize == k4ByteSize || memSize == k8ByteSize) && (prevMemSize == k4ByteSize || prevMemSize == k8ByteSize) && + (diffVal == k8BitSize) && (curDestOpnd.GetValidBitsNum() == memSize * k8BitSize) && + (prevDestOpnd.GetValidBitsNum() == prevMemSize * k8BitSize)) { + return true; } + return false; } void CombineContiLoadAndStorePattern::RemoveInsnAndKeepComment(BB &bb, Insn &insn, Insn &prevInsn) const @@ -3163,28 +4291,200 @@ void EliminateSpecifcUXTAArch64::Run(BB &bb, Insn &insn) } } -bool FmovRegPattern::CheckCondition(Insn &insn) +bool EliminateSpecifcSXTPattern::CheckCondition(Insn &insn) { - nextInsn = insn.GetNextMachineInsn(); - if (nextInsn == nullptr) { - return false; - } - if (&insn == insn.GetBB()->GetFirstInsn()) { - return false; + BB *bb = insn.GetBB(); + if (bb->GetFirstMachineInsn() == &insn) { + BB *prevBB = bb->GetPrev(); + if (prevBB != nullptr && (bb->GetPreds().size() == 1) && (*(bb->GetPreds().cbegin()) == prevBB)) { + prevInsn = prevBB->GetLastMachineInsn(); + } + } else { + prevInsn = insn.GetPreviousMachineInsn(); } - prevInsn = insn.GetPrev(); - auto &curSrcRegOpnd = static_cast(insn.GetOperand(kInsnSecondOpnd)); - auto &prevSrcRegOpnd = static_cast(prevInsn->GetOperand(kInsnSecondOpnd)); - /* same src freg */ - if (curSrcRegOpnd.GetRegisterNumber() != prevSrcRegOpnd.GetRegisterNumber()) { + if (prevInsn == nullptr) { return false; } return true; } -void FmovRegPattern::Run(BB &bb, Insn &insn) +void EliminateSpecifcSXTPattern::Run(BB &bb, Insn &insn) { - if (!CheckCondition(insn)) { + MOperator thisMop = insn.GetMachineOpcode(); + auto ®Opnd0 = static_cast(insn.GetOperand(kInsnFirstOpnd)); + auto ®Opnd1 = static_cast(insn.GetOperand(kInsnSecondOpnd)); + if (regOpnd0.GetRegisterNumber() == regOpnd1.GetRegisterNumber() && prevInsn->IsMachineInstruction()) { + if (prevInsn->GetMachineOpcode() == MOP_wmovri32 || prevInsn->GetMachineOpcode() == MOP_xmovri64) { + auto &dstMovOpnd = static_cast(prevInsn->GetOperand(kInsnFirstOpnd)); + if (dstMovOpnd.GetRegisterNumber() != regOpnd1.GetRegisterNumber()) { + return; + } + Operand &opnd = prevInsn->GetOperand(kInsnSecondOpnd); + if (opnd.IsIntImmediate()) { + auto &immOpnd = static_cast(opnd); + int64 value = immOpnd.GetValue(); + if (thisMop == MOP_xsxtb32) { + /* value should in range between -127 and 127 */ + if (value >= static_cast(0xFFFFFFFFFFFFFF80) && value <= 0x7F && + immOpnd.IsSingleInstructionMovable(regOpnd0.GetSize())) { + bb.RemoveInsn(insn); + optSuccess = true; + return; + } + } else if (thisMop == MOP_xsxth32) { + /* value should in range between -32678 and 32678 */ + if (value >= static_cast(0xFFFFFFFFFFFF8000) && value <= 0x7FFF && + immOpnd.IsSingleInstructionMovable(regOpnd0.GetSize())) { + bb.RemoveInsn(insn); + optSuccess = true; + return; + } + } else { + uint64 flag = 0xFFFFFFFFFFFFFF80; /* initialize the flag with fifty-nine 1s at top */ + if (thisMop == MOP_xsxth64) { + flag = 0xFFFFFFFFFFFF8000; /* specify the flag with forty-nine 1s at top in this case */ + } else if (thisMop == MOP_xsxtw64) { + flag = 0xFFFFFFFF80000000; /* specify the flag with thirty-three 1s at top in this case */ + } + if ((static_cast(value) & flag) == 0 && + immOpnd.IsSingleInstructionMovable(regOpnd0.GetSize())) { + auto *aarch64CGFunc = static_cast(cgFunc); + RegOperand &dstOpnd = aarch64CGFunc->GetOrCreatePhysicalRegisterOperand( + static_cast(dstMovOpnd.GetRegisterNumber()), k64BitSize, + dstMovOpnd.GetRegisterType()); + prevInsn->SetOperand(kInsnFirstOpnd, dstOpnd); + prevInsn->SetMOP(AArch64CG::kMd[MOP_xmovri64]); + bb.RemoveInsn(insn); + optSuccess = true; + return; + } + } + } + } + } +} + +bool EliminateSpecifcUXTPattern::CheckCondition(Insn &insn) +{ + BB *bb = insn.GetBB(); + if (bb->GetFirstMachineInsn() == &insn) { + BB *prevBB = bb->GetPrev(); + if (prevBB != nullptr && (bb->GetPreds().size() == 1) && (*(bb->GetPreds().cbegin()) == prevBB)) { + prevInsn = prevBB->GetLastMachineInsn(); + } + } else { + prevInsn = insn.GetPreviousMachineInsn(); + } + if (prevInsn == nullptr) { + return false; + } + return true; +} + +void EliminateSpecifcUXTPattern::Run(BB &bb, Insn &insn) +{ + if (!CheckCondition(insn)) { + return; + } + MOperator thisMop = insn.GetMachineOpcode(); + auto ®Opnd0 = static_cast(insn.GetOperand(kInsnFirstOpnd)); + auto ®Opnd1 = static_cast(insn.GetOperand(kInsnSecondOpnd)); + if (&insn == bb.GetFirstMachineInsn() || regOpnd0.GetRegisterNumber() != regOpnd1.GetRegisterNumber() || + !prevInsn->IsMachineInstruction()) { + return; + } + if (cgFunc->GetMirModule().GetSrcLang() == kSrcLangC && prevInsn->IsCall() && prevInsn->GetIsCallReturnSigned()) { + return; + } + if (thisMop == MOP_xuxtb32) { + if (prevInsn->GetMachineOpcode() == MOP_wmovri32 || prevInsn->GetMachineOpcode() == MOP_xmovri64) { + auto &dstMovOpnd = static_cast(prevInsn->GetOperand(kInsnFirstOpnd)); + if (!IsSameRegisterOperation(dstMovOpnd, regOpnd1, regOpnd0)) { + return; + } + Operand &opnd = prevInsn->GetOperand(kInsnSecondOpnd); + if (opnd.IsIntImmediate()) { + auto &immOpnd = static_cast(opnd); + int64 value = immOpnd.GetValue(); + /* check the top 56 bits of value */ + if ((static_cast(value) & 0xFFFFFFFFFFFFFF00) == 0) { + bb.RemoveInsn(insn); + optSuccess = true; + return; + } + } + } else if (prevInsn->GetMachineOpcode() == MOP_wldrb) { + auto &dstOpnd = static_cast(prevInsn->GetOperand(kInsnFirstOpnd)); + if (dstOpnd.GetRegisterNumber() != regOpnd1.GetRegisterNumber()) { + return; + } + bb.RemoveInsn(insn); + optSuccess = true; + return; + } + } else if (thisMop == MOP_xuxth32) { + if (prevInsn->GetMachineOpcode() == MOP_wmovri32 || prevInsn->GetMachineOpcode() == MOP_xmovri64) { + auto &dstMovOpnd = static_cast(prevInsn->GetOperand(kInsnFirstOpnd)); + if (!IsSameRegisterOperation(dstMovOpnd, regOpnd1, regOpnd0)) { + return; + } + Operand &opnd = prevInsn->GetOperand(kInsnSecondOpnd); + if (opnd.IsIntImmediate()) { + auto &immOpnd = static_cast(opnd); + int64 value = immOpnd.GetValue(); + if ((static_cast(value) & 0xFFFFFFFFFFFF0000) == 0) { + bb.RemoveInsn(insn); + optSuccess = true; + return; + } + } + } else if (prevInsn->GetMachineOpcode() == MOP_wldrh) { + auto &dstOpnd = static_cast(prevInsn->GetOperand(kInsnFirstOpnd)); + if (dstOpnd.GetRegisterNumber() != regOpnd1.GetRegisterNumber()) { + return; + } + bb.RemoveInsn(insn); + optSuccess = true; + return; + } + } else { + if (prevInsn->GetMachineOpcode() == MOP_wmovri32 || prevInsn->GetMachineOpcode() == MOP_wldrsb || + prevInsn->GetMachineOpcode() == MOP_wldrb || prevInsn->GetMachineOpcode() == MOP_wldrsh || + prevInsn->GetMachineOpcode() == MOP_wldrh || prevInsn->GetMachineOpcode() == MOP_wldr) { + auto &dstOpnd = static_cast(prevInsn->GetOperand(kInsnFirstOpnd)); + if (!IsSameRegisterOperation(dstOpnd, regOpnd1, regOpnd0)) { + return; + } + /* 32-bit ldr does zero-extension by default, so this conversion can be skipped */ + bb.RemoveInsn(insn); + optSuccess = true; + return; + } + } +} + +bool FmovRegPattern::CheckCondition(Insn &insn) +{ + nextInsn = insn.GetNextMachineInsn(); + if (nextInsn == nullptr) { + return false; + } + if (&insn == insn.GetBB()->GetFirstMachineInsn()) { + return false; + } + prevInsn = insn.GetPrev(); + auto &curSrcRegOpnd = static_cast(insn.GetOperand(kInsnSecondOpnd)); + auto &prevSrcRegOpnd = static_cast(prevInsn->GetOperand(kInsnSecondOpnd)); + /* same src freg */ + if (curSrcRegOpnd.GetRegisterNumber() != prevSrcRegOpnd.GetRegisterNumber()) { + return false; + } + return true; +} + +void FmovRegPattern::Run(BB &bb, Insn &insn) +{ + if (!CheckCondition(insn)) { return; } MOperator thisMop = insn.GetMachineOpcode(); @@ -3259,8 +4559,13 @@ bool SbfxOptPattern::CheckCondition(Insn &insn) return false; } auto &curDstRegOpnd = static_cast(insn.GetOperand(kInsnFirstOpnd)); + auto &lsb = static_cast(insn.GetOperand(kInsnThirdOpnd)); + auto &width = static_cast(insn.GetOperand(kInsnFourthOpnd)); + if (lsb.GetValue() != 0 || width.GetValue() < k32BitSize) { + return false; + } uint32 opndNum = nextInsn->GetOperandSize(); - const InsnDesc *md = insn.GetDesc(); + const InsnDesc *md = nextInsn->GetDesc(); for (uint32 opndIdx = 0; opndIdx < opndNum; ++opndIdx) { Operand &opnd = nextInsn->GetOperand(opndIdx); /* Check if it is a source operand. */ @@ -3270,7 +4575,7 @@ bool SbfxOptPattern::CheckCondition(Insn &insn) auto ® = static_cast(opnd); auto *regProp = md->opndMD[opndIdx]; if (reg.GetRegisterNumber() == curDstRegOpnd.GetRegisterNumber()) { - if (reg.GetSize() != k32BitSize) { + if (nextInsn->GetOperandSize(opndIdx) != k32BitSize) { return false; } if (regProp->IsDef()) { @@ -3416,6 +4721,41 @@ void CbnzToCbzPattern::Run(BB &bb, Insn &insn) targetBB->PushBackPreds(*nextBB); } +bool ContiLDRorSTRToSameMEMPattern::HasImplicitSizeUse(const Insn &insn) const +{ + if (insn.GetOperandSize(kInsnFirstOpnd) != prevInsn->GetOperandSize(kInsnFirstOpnd)) { + return true; + } + if (loadAfterStore) { + // To avoid the optimization as following: + // str w10, [sp, #8] + // ldr w10, [sp, #8] ---\--> can not be removed + // ... + // str x10, [x1, #16] + auto &defOpnd = static_cast(insn.GetOperand(kInsnFirstOpnd)); + for (Insn *cursor = insn.GetNext(); cursor != nullptr; cursor = cursor->GetNext()) { + if (!cursor->IsMachineInstruction()) { + continue; + } + uint32 opndNum = cursor->GetOperandSize(); + for (uint32 i = 0; i < opndNum; ++i) { + if (cursor->OpndIsDef(i)) { + continue; + } + if (!cursor->GetOperand(i).IsRegister()) { + continue; + } + auto &useOpnd = static_cast(cursor->GetOperand(i)); + if (useOpnd.GetRegisterNumber() == defOpnd.GetRegisterNumber() && + insn.GetOperandSize(kInsnFirstOpnd) != cursor->GetOperandSize(i)) { + return true; + } + } + } + } + return false; +} + void CsetCbzToBeqOptAArch64::Run(BB &bb, Insn &insn) { Insn *insn1 = insn.GetPreviousMachineInsn(); @@ -3492,7 +4832,8 @@ MOperator CsetCbzToBeqOptAArch64::SelectMOperator(ConditionCode condCode, bool i bool ContiLDRorSTRToSameMEMPattern::CheckCondition(Insn &insn) { prevInsn = insn.GetPrev(); - while (prevInsn != nullptr && !prevInsn->GetMachineOpcode() && prevInsn != insn.GetBB()->GetFirstInsn()) { + while (prevInsn != nullptr && (prevInsn->GetMachineOpcode() == 0 || !prevInsn->IsMachineInstruction()) && + prevInsn != insn.GetBB()->GetFirstMachineInsn()) { prevInsn = prevInsn->GetPrev(); } if (!insn.IsMachineInstruction() || prevInsn == nullptr) { @@ -3519,6 +4860,9 @@ bool ContiLDRorSTRToSameMEMPattern::CheckCondition(Insn &insn) if (!loadAfterStore && !loadAfterLoad) { return false; } + if (HasImplicitSizeUse(insn)) { + return false; + } DEBUG_ASSERT(insn.GetOperand(kInsnSecondOpnd).IsMemoryAccessOperand(), "expects mem operands"); DEBUG_ASSERT(prevInsn->GetOperand(kInsnSecondOpnd).IsMemoryAccessOperand(), "expects mem operands"); return true; @@ -3564,14 +4908,14 @@ void ContiLDRorSTRToSameMEMPattern::Run(BB &bb, Insn &insn) } if (loadAfterStore && reg1.GetRegisterNumber() != reg2.GetRegisterNumber()) { /* replace it with mov */ - MOperator newOp = MOP_wmovrr; + MOperator newOp = MOP_undef; if (reg1.GetRegisterType() == kRegTyInt) { - newOp = (reg1.GetSize() <= k32BitSize) ? MOP_wmovrr : MOP_xmovrr; + newOp = (insn.GetOperandSize(kInsnFirstOpnd) == k32BitSizeInt) ? MOP_wmovrr : MOP_xmovrr; } else if (reg1.GetRegisterType() == kRegTyFloat) { - newOp = (reg1.GetSize() <= k32BitSize) ? MOP_xvmovs : MOP_xvmovd; + newOp = (insn.GetOperandSize(kInsnFirstOpnd) == k32BitSizeInt) ? MOP_xvmovs : MOP_xvmovd; } Insn *nextInsn = insn.GetNext(); - while (nextInsn != nullptr && !nextInsn->GetMachineOpcode() && nextInsn != bb.GetLastInsn()) { + while (nextInsn != nullptr && !nextInsn->IsMachineInstruction() && nextInsn != bb.GetLastMachineInsn()) { nextInsn = nextInsn->GetNext(); } bool moveSameReg = false; @@ -3586,9 +4930,13 @@ void ContiLDRorSTRToSameMEMPattern::Run(BB &bb, Insn &insn) if (!moveSameReg) { bb.InsertInsnAfter(*prevInsn, cgFunc->GetInsnBuilder()->BuildInsn(newOp, reg1, reg2)); } + SetCurrInsn(insn.GetNextMachineInsn()); + optSuccess = true; bb.RemoveInsn(insn); } else if (reg1.GetRegisterNumber() == reg2.GetRegisterNumber() && base1->GetRegisterNumber() != reg2.GetRegisterNumber()) { + SetCurrInsn(insn.GetNextMachineInsn()); + optSuccess = true; bb.RemoveInsn(insn); } } @@ -3682,7 +5030,8 @@ Insn *CselZeroOneToCsetOpt::FindFixedValue(Operand &opnd, BB &bb, Operand *&temp bool alreadyFindCsel = false; bool isRegDefined = false; regno_t regno = static_cast(opnd).GetRegisterNumber(); - FOR_BB_INSNS_REV(defInsn, &bb) { + FOR_BB_INSNS_REV(defInsn, &bb) + { if (!defInsn->IsMachineInstruction() || defInsn->IsBranch()) { continue; } @@ -3727,7 +5076,7 @@ void AndCmpCsetEorCbzOpt::Run(BB &bb, Insn &insn) return; } RegOperand &cmpInsnSecondOpnd = static_cast(cmpInsn->GetOperand(kInsnSecondOpnd)); - ImmOperand &cmpInsnThirdOpnd = static_cast(cmpInsn->GetOperand(kInsnThirdOpnd)); + ImmOperand &cmpInsnThirdOpnd = static_cast(cmpInsn->GetOperand(kInsnThirdOpnd)); if (cmpInsnSecondOpnd.GetRegisterNumber() != andInsnFirstOpnd.GetRegisterNumber() || cmpInsnThirdOpnd.GetValue() != 0) { return; @@ -3785,8 +5134,8 @@ void AddLdrOpt::Run(BB &bb, Insn &insn) if (nextMop != MOP_xldr && nextMop != MOP_wldr) { return; } - RegOperand &insnFirstOpnd = static_cast(insn.GetOperand(kInsnFirstOpnd)); - RegOperand &insnSecondOpnd = static_cast(insn.GetOperand(kInsnSecondOpnd)); + RegOperand &insnFirstOpnd = static_cast(insn.GetOperand(kInsnFirstOpnd)); + RegOperand &insnSecondOpnd = static_cast(insn.GetOperand(kInsnSecondOpnd)); if (insnFirstOpnd.GetRegisterNumber() != insnSecondOpnd.GetRegisterNumber()) { return; } @@ -3798,8 +5147,9 @@ void AddLdrOpt::Run(BB &bb, Insn &insn) memOpnd.GetOffsetImmediate()->GetOffsetValue() != 0) { return; } - MemOperand &newMemOpnd = static_cast(cgFunc)->GetOrCreateMemOpnd(MemOperand::kAddrModeBOrX, - memOpnd.GetSize(), &insnFirstOpnd, &static_cast(insn.GetOperand(kInsnThirdOpnd)), 0, false); + MemOperand &newMemOpnd = static_cast(cgFunc)->GetOrCreateMemOpnd( + MemOperand::kAddrModeBOrX, memOpnd.GetSize(), &insnFirstOpnd, + &static_cast(insn.GetOperand(kInsnThirdOpnd)), 0, false); nextInsn->SetOperand(kInsnSecondOpnd, newMemOpnd); bb.RemoveInsn(insn); } @@ -3818,8 +5168,7 @@ void CsetEorOpt::Run(BB &bb, Insn &insn) RegOperand &eorFirstOpnd = static_cast(nextInsn->GetOperand(kInsnFirstOpnd)); RegOperand &eorSecondOpnd = static_cast(nextInsn->GetOperand(kInsnSecondOpnd)); ImmOperand &eorThirdOpnd = static_cast(nextInsn->GetOperand(kInsnThirdOpnd)); - if (eorThirdOpnd.GetValue() != 1 || - eorFirstOpnd.GetRegisterNumber() != eorSecondOpnd.GetRegisterNumber() || + if (eorThirdOpnd.GetValue() != 1 || eorFirstOpnd.GetRegisterNumber() != eorSecondOpnd.GetRegisterNumber() || csetFirstOpnd.GetRegisterNumber() != eorFirstOpnd.GetRegisterNumber()) { return; } @@ -3858,8 +5207,8 @@ void MoveCmpOpt::Run(BB &bb, Insn &insn) return; } MOperator cmpOpCode = (cmpThirdOpnd.GetSize() == k64BitSize) ? MOP_xcmpri : MOP_wcmpri; - Insn &newCmpInsn = cgFunc->GetInsnBuilder()->BuildInsn( - cmpOpCode, nextInsn->GetOperand(kInsnFirstOpnd), nextInsn->GetOperand(kInsnSecondOpnd), immOpnd); + Insn &newCmpInsn = cgFunc->GetInsnBuilder()->BuildInsn(cmpOpCode, nextInsn->GetOperand(kInsnFirstOpnd), + nextInsn->GetOperand(kInsnSecondOpnd), immOpnd); bb.ReplaceInsn(*nextInsn, newCmpInsn); if (!IfOperandIsLiveAfterInsn(movFirstOpnd, newCmpInsn)) { bb.RemoveInsn(insn); @@ -3922,7 +5271,7 @@ bool ReplaceDivToMultiPattern::CheckCondition(Insn &insn) } MOperator prevMop = prevInsn->GetMachineOpcode(); MOperator prePrevMop = prePrevInsn->GetMachineOpcode(); - if (prevMop && (prevMop == MOP_wmovkri16) && prePrevMop && (prePrevMop == MOP_wmovri32)) { + if ((prevMop > 0) && (prevMop == MOP_wmovkri16) && (prePrevMop > 0) && (prePrevMop == MOP_wmovri32)) { return true; } return false; @@ -3934,8 +5283,11 @@ void ReplaceDivToMultiPattern::Run(BB &bb, Insn &insn) auto &sdivOpnd1 = static_cast(insn.GetOperand(kInsnSecondOpnd)); auto &sdivOpnd2 = static_cast(insn.GetOperand(kInsnThirdOpnd)); /* Check if dest operand of insn is idential with register of prevInsn and prePrevInsn. */ - if ((&(prevInsn->GetOperand(kInsnFirstOpnd)) != &sdivOpnd2) || - (&(prePrevInsn->GetOperand(kInsnFirstOpnd)) != &sdivOpnd2)) { + auto &prevReg = prevInsn->GetOperand(kInsnFirstOpnd); + auto &prePrevReg = prePrevInsn->GetOperand(kInsnFirstOpnd); + if (!prevReg.IsRegister() || !prePrevReg.IsRegister() || + static_cast(prevReg).GetRegisterNumber() != sdivOpnd2.GetRegisterNumber() || + static_cast(prePrevReg).GetRegisterNumber() != sdivOpnd2.GetRegisterNumber()) { return; } auto &prevLsl = static_cast(prevInsn->GetOperand(kInsnThirdOpnd)); @@ -3948,7 +5300,8 @@ void ReplaceDivToMultiPattern::Run(BB &bb, Insn &insn) * expect the immediate value of first mov is 0x086A0 which matches 0x186A0 * because 0x10000 is ignored in 32 bits register */ - if ((prevImmOpnd.GetValue() != 1) || (prePrevImmOpnd.GetValue() != 0x86a0)) { + constexpr uint32 immOpndBoundary = 34464; + if ((prevImmOpnd.GetValue() != 1) || (prePrevImmOpnd.GetValue() != immOpndBoundary)) { return; } auto *aarch64CGFunc = static_cast(cgFunc); @@ -4091,7 +5444,7 @@ void AndCmpBranchesToCsetAArch64::Run(BB &bb, Insn &insn) } /* andImmVal is n power of 2 */ - int n = logValueAtBase2(andImmVal); + int n = LogValueAtBase2(andImmVal); if (n < 0) { return; } @@ -4115,57 +5468,6 @@ void AndCmpBranchesToCsetAArch64::Run(BB &bb, Insn &insn) } } -void AndCmpBranchesToTstAArch64::Run(BB &bb, Insn &insn) -{ - /* nextInsn must be "cmp" insn */ - Insn *nextInsn = insn.GetNextMachineInsn(); - if (nextInsn == nullptr || - (nextInsn->GetMachineOpcode() != MOP_wcmpri && nextInsn->GetMachineOpcode() != MOP_xcmpri)) { - return; - } - /* nextNextInsn must be "beq" or "bne" insn */ - Insn *nextNextInsn = nextInsn->GetNextMachineInsn(); - if (nextNextInsn == nullptr || - (nextNextInsn->GetMachineOpcode() != MOP_beq && nextNextInsn->GetMachineOpcode() != MOP_bne)) { - return; - } - auto &andRegOp = static_cast(insn.GetOperand(kInsnFirstOpnd)); - regno_t andRegNO1 = andRegOp.GetRegisterNumber(); - auto &cmpRegOp2 = static_cast(nextInsn->GetOperand(kInsnSecondOpnd)); - regno_t cmpRegNO2 = cmpRegOp2.GetRegisterNumber(); - if (andRegNO1 != cmpRegNO2) { - return; - } - /* If the reg will be used later, we shouldn't optimize the and insn here */ - if (IfOperandIsLiveAfterInsn(andRegOp, *nextInsn)) { - return; - } - Operand &immOpnd = nextInsn->GetOperand(kInsnThirdOpnd); - DEBUG_ASSERT(immOpnd.IsIntImmediate(), "expects ImmOperand"); - auto &defConst = static_cast(immOpnd); - int64 defConstValue = defConst.GetValue(); - if (defConstValue != 0) { - return; - } - /* build tst insn */ - Operand &andOpnd3 = insn.GetOperand(kInsnThirdOpnd); - auto &andRegOp2 = static_cast(insn.GetOperand(kInsnSecondOpnd)); - MOperator newOp = MOP_undef; - if (andOpnd3.IsRegister()) { - newOp = (andRegOp2.GetSize() <= k32BitSize) ? MOP_wtstrr : MOP_xtstrr; - } else { - newOp = (andRegOp2.GetSize() <= k32BitSize) ? MOP_wtstri32 : MOP_xtstri64; - } - Operand &rflag = static_cast(&cgFunc)->GetOrCreateRflag(); - Insn &newInsn = cgFunc.GetInsnBuilder()->BuildInsn(newOp, rflag, andRegOp2, andOpnd3); - if (CGOptions::DoCGSSA() && CGOptions::GetInstance().GetOptimizeLevel() < CGOptions::kLevel0) { - CHECK_FATAL(false, "check this case in ssa opt"); - } - bb.InsertInsnAfter(*nextInsn, newInsn); - bb.RemoveInsn(insn); - bb.RemoveInsn(*nextInsn); -} - void AndCbzBranchesToTstAArch64::Run(BB &bb, Insn &insn) { /* nextInsn must be "cbz" or "cbnz" insn */ @@ -4212,33 +5514,261 @@ void AndCbzBranchesToTstAArch64::Run(BB &bb, Insn &insn) bb.ReplaceInsn(*nextInsn, newInsnJmp); } -void ZeroCmpBranchesAArch64::Run(BB &bb, Insn &insn) +bool AndCmpBranchesToCsetPattern::CheckCondition(Insn &insn) { - Insn *prevInsn = insn.GetPreviousMachineInsn(); - if (!insn.IsBranch() || insn.GetOperandSize() <= kInsnSecondOpnd || prevInsn == nullptr) { - return; + /* prevInsn must be "cmp" insn */ + auto &ccReg = static_cast(insn.GetOperand(kInsnThirdOpnd)); + prevCmpInsn = ssaInfo->GetDefInsn(ccReg); + if (prevCmpInsn == nullptr) { + return false; } - if (!insn.GetOperand(kInsnSecondOpnd).IsLabel()) { - return; + MOperator prevCmpMop = prevCmpInsn->GetMachineOpcode(); + if (prevCmpMop != MOP_wcmpri && prevCmpMop != MOP_xcmpri) { + return false; } - LabelOperand *label = &static_cast(insn.GetOperand(kInsnSecondOpnd)); - RegOperand *regOpnd = nullptr; - RegOperand *reg0 = nullptr; - RegOperand *reg1 = nullptr; - MOperator newOp = MOP_undef; - ImmOperand *imm = nullptr; - switch (prevInsn->GetMachineOpcode()) { - case MOP_wcmpri: - case MOP_xcmpri: { - regOpnd = &static_cast(prevInsn->GetOperand(kInsnSecondOpnd)); - imm = &static_cast(prevInsn->GetOperand(kInsnThirdOpnd)); - if (imm->GetValue() != 0) { - return; - } - if (insn.GetMachineOpcode() == MOP_bge) { - newOp = (regOpnd->GetSize() <= k32BitSize) ? MOP_wtbz : MOP_xtbz; - } else if (insn.GetMachineOpcode() == MOP_blt) { - newOp = (regOpnd->GetSize() <= k32BitSize) ? MOP_wtbnz : MOP_xtbnz; + /* prevPrevInsn must be "and" insn */ + auto &cmpUseReg = static_cast(prevCmpInsn->GetOperand(kInsnSecondOpnd)); + prevAndInsn = ssaInfo->GetDefInsn(cmpUseReg); + if (prevAndInsn == nullptr) { + return false; + } + MOperator prevAndMop = prevAndInsn->GetMachineOpcode(); + if (prevAndMop != MOP_wandrri12 && prevAndMop != MOP_xandrri13) { + return false; + } + CHECK_FATAL(prevAndInsn->GetOperand(kInsnFirstOpnd).GetSize() == prevCmpInsn->GetOperand(kInsnSecondOpnd).GetSize(), + "def-use reg size must be same based-on ssa"); + auto &csetCond = static_cast(insn.GetOperand(kInsnSecondOpnd)); + auto &cmpImm = static_cast(prevCmpInsn->GetOperand(kInsnThirdOpnd)); + int64 cmpImmVal = cmpImm.GetValue(); + auto &andImm = static_cast(prevAndInsn->GetOperand(kInsnThirdOpnd)); + int64 andImmVal = andImm.GetValue(); + if ((csetCond.GetCode() == CC_EQ && cmpImmVal == andImmVal) || (csetCond.GetCode() == CC_NE && cmpImmVal == 0)) { + /* guaranteed unique point of use */ + auto &flagReg = static_cast(prevCmpInsn->GetOperand(kInsnFirstOpnd)); + InsnSet cmpFirstUseSet = GetAllUseInsn(flagReg); + if (cmpFirstUseSet.size() > 1) { + return false; + } + /* guaranteed unique point of use */ + auto &prevInsnSecondReg = prevCmpInsn->GetOperand(kInsnSecondOpnd); + InsnSet cmpSecondUseSet = GetAllUseInsn(static_cast(prevInsnSecondReg)); + if (cmpSecondUseSet.size() > 1) { + return false; + } + return true; + } + return false; +} + +void AndCmpBranchesToCsetPattern::Run(BB &bb, Insn &insn) +{ + if (!CheckCondition(insn)) { + return; + } + RegOperand *dstOpnd = nullptr; + RegOperand *srcOpnd = nullptr; + auto &andImm = static_cast(prevAndInsn->GetOperand(kInsnThirdOpnd)); + int64 andImmVal = andImm.GetValue(); + if (andImmVal == 1) { + /* Method 1: Delete cmp and cset, and replace cset with and. */ + dstOpnd = &static_cast(prevAndInsn->GetOperand(kInsnFirstOpnd)); + srcOpnd = &static_cast(insn.GetOperand(kInsnFirstOpnd)); + if (dstOpnd->IsPhysicalRegister() || srcOpnd->IsPhysicalRegister()) { + return; + } + VRegVersion *dstVersion = ssaInfo->FindSSAVersion(dstOpnd->GetRegisterNumber()); + VRegVersion *srcVersion = ssaInfo->FindSSAVersion(srcOpnd->GetRegisterNumber()); + CHECK_FATAL(dstVersion != nullptr, "get dstVersion failed"); + CHECK_FATAL(srcVersion != nullptr, "get srcVersion failed"); + /* Ensure that there is no use point */ + auto &insnDefReg = static_cast(insn.GetOperand(kInsnFirstOpnd)); + InsnSet csetFirstUseSet = GetAllUseInsn(insnDefReg); + if (csetFirstUseSet.size() < 1) { + return; + } + /* update ssa info */ + ssaInfo->ReplaceAllUse(srcVersion, dstVersion); + optSuccess = true; + /* dump pattern info */ + if (CG_PEEP_DUMP) { + std::vector prevs; + prevs.emplace_back(prevAndInsn); + prevs.emplace_back(prevCmpInsn); + DumpAfterPattern(prevs, &insn, prevAndInsn); + } + } else { + /* andImmVal is n power of 2 */ + int64 n = GetLogValueAtBase2(andImmVal); + if (n < 0) { + return; + } + /* Method 2: ubfx replaces cset. */ + /* create ubfx insn */ + auto &csetReg = static_cast(insn.GetOperand(kInsnFirstOpnd)); + MOperator ubfxOp = (csetReg.GetSize() <= k32BitSize) ? MOP_wubfxrri5i5 : MOP_xubfxrri6i6; + if (ubfxOp == MOP_wubfxrri5i5 && static_cast(n) >= k32BitSize) { + return; + } + auto &dstReg = static_cast(csetReg); + auto &prevAndInsnSecondReg = prevAndInsn->GetOperand(kInsnSecondOpnd); + auto &srcReg = static_cast(prevAndInsnSecondReg); + auto *aarch64CGFunc = static_cast(cgFunc); + ImmOperand &bitPos = aarch64CGFunc->CreateImmOperand(n, k8BitSize, false); + ImmOperand &bitSize = aarch64CGFunc->CreateImmOperand(1, k8BitSize, false); + Insn &ubfxInsn = cgFunc->GetInsnBuilder()->BuildInsn(ubfxOp, dstReg, srcReg, bitPos, bitSize); + bb.ReplaceInsn(insn, ubfxInsn); + /* update ssa info */ + ssaInfo->ReplaceInsn(insn, ubfxInsn); + optSuccess = true; + SetCurrInsn(&ubfxInsn); + /* dump pattern info */ + if (CG_PEEP_DUMP) { + std::vector prevs; + prevs.emplace_back(prevAndInsn); + prevs.emplace_back(prevCmpInsn); + DumpAfterPattern(prevs, &insn, &ubfxInsn); + } + } +} + +void AndCmpBranchesToTstAArch64::Run(BB &bb, Insn &insn) +{ + /* nextInsn must be "cmp" insn */ + Insn *nextInsn = insn.GetNextMachineInsn(); + if (nextInsn == nullptr || + (nextInsn->GetMachineOpcode() != MOP_wcmpri && nextInsn->GetMachineOpcode() != MOP_xcmpri)) { + return; + } + /* nextNextInsn must be "beq" or "bne" insn */ + Insn *nextNextInsn = nextInsn->GetNextMachineInsn(); + if (nextNextInsn == nullptr || + (nextNextInsn->GetMachineOpcode() != MOP_beq && nextNextInsn->GetMachineOpcode() != MOP_bne)) { + return; + } + auto &andRegOp = static_cast(insn.GetOperand(kInsnFirstOpnd)); + regno_t andRegNO1 = andRegOp.GetRegisterNumber(); + auto &cmpRegOp2 = static_cast(nextInsn->GetOperand(kInsnSecondOpnd)); + regno_t cmpRegNO2 = cmpRegOp2.GetRegisterNumber(); + if (andRegNO1 != cmpRegNO2) { + return; + } + /* If the reg will be used later, we shouldn't optimize the and insn here */ + if (IfOperandIsLiveAfterInsn(andRegOp, *nextInsn)) { + return; + } + Operand &immOpnd = nextInsn->GetOperand(kInsnThirdOpnd); + DEBUG_ASSERT(immOpnd.IsIntImmediate(), "expects ImmOperand"); + auto &defConst = static_cast(immOpnd); + int64 defConstValue = defConst.GetValue(); + if (defConstValue != 0) { + return; + } + /* build tst insn */ + Operand &andOpnd3 = insn.GetOperand(kInsnThirdOpnd); + auto &andRegOp2 = static_cast(insn.GetOperand(kInsnSecondOpnd)); + MOperator newOp = MOP_undef; + if (andOpnd3.IsRegister()) { + newOp = (andRegOp2.GetSize() <= k32BitSize) ? MOP_wtstrr : MOP_xtstrr; + } else { + newOp = (andRegOp2.GetSize() <= k32BitSize) ? MOP_wtstri32 : MOP_xtstri64; + } + Operand &rflag = static_cast(&cgFunc)->GetOrCreateRflag(); + Insn &newInsn = cgFunc.GetInsnBuilder()->BuildInsn(newOp, rflag, andRegOp2, andOpnd3); + if (CGOptions::DoCGSSA() && CGOptions::GetInstance().GetOptimizeLevel() < CGOptions::kLevel0) { + CHECK_FATAL(false, "check this case in ssa opt"); + } + bb.InsertInsnAfter(*nextInsn, newInsn); + bb.RemoveInsn(insn); + bb.RemoveInsn(*nextInsn); +} + +bool AndCbzBranchesToTstPattern::CheckCondition(Insn &insn) +{ + /* nextInsn must be "cbz" or "cbnz" insn */ + Insn *nextInsn = insn.GetNextMachineInsn(); + if (nextInsn == nullptr || (nextInsn->GetMachineOpcode() != MOP_wcbz && nextInsn->GetMachineOpcode() != MOP_xcbz)) { + return false; + } + auto &andRegOp1 = static_cast(insn.GetOperand(kInsnFirstOpnd)); + regno_t andRegNo1 = andRegOp1.GetRegisterNumber(); + auto &cbzRegOp1 = static_cast(nextInsn->GetOperand(kInsnFirstOpnd)); + regno_t cbzRegNo1 = cbzRegOp1.GetRegisterNumber(); + if (andRegNo1 != cbzRegNo1) { + return false; + } + /* If the reg will be used later, we shouldn't optimize the and insn here */ + if (IfOperandIsLiveAfterInsn(andRegOp1, *nextInsn)) { + return false; + } + auto &andRegOp2 = static_cast(insn.GetOperand(kInsnSecondOpnd)); + Operand &andOpnd3 = insn.GetOperand(kInsnThirdOpnd); + if (andOpnd3.IsImmediate() && !static_cast(andOpnd3).IsBitmaskImmediate(andRegOp2.GetSize())) { + return false; + } + /* avoid redefine cc-reg */ + if (static_cast(cgFunc)->GetRflag() != nullptr) { + return false; + } + return true; +} + +void AndCbzBranchesToTstPattern::Run(BB &bb, Insn &insn) +{ + if (!CheckCondition(insn)) { + return; + } + Insn *nextInsn = insn.GetNextMachineInsn(); + CHECK_NULL_FATAL(nextInsn); + /* build tst insn */ + auto &andRegOp2 = static_cast(insn.GetOperand(kInsnSecondOpnd)); + Operand &andOpnd3 = insn.GetOperand(kInsnThirdOpnd); + MOperator newTstOp = MOP_undef; + if (andOpnd3.IsRegister()) { + newTstOp = (insn.GetMachineOpcode() == MOP_wandrrr) ? MOP_wtstrr : MOP_xtstrr; + } else { + newTstOp = (insn.GetMachineOpcode() == MOP_wandrri12) ? MOP_wtstri32 : MOP_xtstri64; + } + Operand &rflag = static_cast(cgFunc)->GetOrCreateRflag(); + Insn &newInsnTst = cgFunc->GetInsnBuilder()->BuildInsn(newTstOp, rflag, andRegOp2, andOpnd3); + + /* build beq insn */ + MOperator opCode = nextInsn->GetMachineOpcode(); + bool reverse = (opCode == MOP_xcbz || opCode == MOP_wcbz); + auto &label = static_cast(nextInsn->GetOperand(kInsnSecondOpnd)); + MOperator jmpOperator = reverse ? MOP_beq : MOP_bne; + Insn &newInsnJmp = cgFunc->GetInsnBuilder()->BuildInsn(jmpOperator, rflag, label); + bb.ReplaceInsn(insn, newInsnTst); + bb.ReplaceInsn(*nextInsn, newInsnJmp); +} + +void ZeroCmpBranchesAArch64::Run(BB &bb, Insn &insn) +{ + Insn *prevInsn = insn.GetPreviousMachineInsn(); + if (!insn.IsBranch() || insn.GetOperandSize() <= kInsnSecondOpnd || prevInsn == nullptr) { + return; + } + if (!insn.GetOperand(kInsnSecondOpnd).IsLabel()) { + return; + } + LabelOperand *label = &static_cast(insn.GetOperand(kInsnSecondOpnd)); + RegOperand *regOpnd = nullptr; + RegOperand *reg0 = nullptr; + RegOperand *reg1 = nullptr; + MOperator newOp = MOP_undef; + ImmOperand *imm = nullptr; + switch (prevInsn->GetMachineOpcode()) { + case MOP_wcmpri: + case MOP_xcmpri: { + regOpnd = &static_cast(prevInsn->GetOperand(kInsnSecondOpnd)); + imm = &static_cast(prevInsn->GetOperand(kInsnThirdOpnd)); + if (imm->GetValue() != 0) { + return; + } + if (insn.GetMachineOpcode() == MOP_bge) { + newOp = (regOpnd->GetSize() <= k32BitSize) ? MOP_wtbz : MOP_xtbz; + } else if (insn.GetMachineOpcode() == MOP_blt) { + newOp = (regOpnd->GetSize() <= k32BitSize) ? MOP_wtbnz : MOP_xtbnz; } else { return; } @@ -4319,7 +5849,7 @@ void ElimDuplicateExtensionAArch64::Run(BB &bb, Insn &insn) [[clang::fallthrough]]; case MOP_xsxtb64: table = sextMopTable; - index = 0; // 0 is index of MOP_xsxtb32 in table sextMopTable + index = 0; // 0 is index of MOP_xsxtb32 in table sextMopTable upper = kSizeOfSextMopTable; break; case MOP_xsxth32: @@ -4327,29 +5857,29 @@ void ElimDuplicateExtensionAArch64::Run(BB &bb, Insn &insn) [[clang::fallthrough]]; case MOP_xsxth64: table = sextMopTable; - index = 2; // 2 is index of MOP_xsxth32 in table sextMopTable + index = 2; // 2 is index of MOP_xsxth32 in table sextMopTable upper = kSizeOfSextMopTable; break; case MOP_xsxtw64: table = sextMopTable; - index = 4; // 4 is index of MOP_xsxtw64 in table sextMopTable + index = 4; // 4 is index of MOP_xsxtw64 in table sextMopTable upper = kSizeOfSextMopTable; break; case MOP_xuxtb32: is32bits = true; table = uextMopTable; - index = 0; // 0 is index of MOP_xuxtb32 in table uextMopTable + index = 0; // 0 is index of MOP_xuxtb32 in table uextMopTable upper = kSizeOfUextMopTable; break; case MOP_xuxth32: is32bits = true; table = uextMopTable; - index = 1; // 1 is index of MOP_xuxth32 in table uextMopTable + index = 1; // 1 is index of MOP_xuxth32 in table uextMopTable upper = kSizeOfUextMopTable; break; case MOP_xuxtw64: table = uextMopTable; - index = 2; // 2 is index of MOP_xuxtw64 in table uextMopTable + index = 2; // 2 is index of MOP_xuxtw64 in table uextMopTable upper = kSizeOfUextMopTable; break; default: @@ -4584,7 +6114,7 @@ void CmpCsetAArch64::Run(BB &bb, Insn &insn) * according to cbz, return true if insn is cbz or cbnz and the first operand of cbz(cbnz) is same as input * operand */ -bool DeleteMovAfterCbzOrCbnzAArch64::PredBBCheck(BB &bb, bool checkCbz, const Operand &opnd) const +bool DeleteMovAfterCbzOrCbnzAArch64::PredBBCheck(BB &bb, bool checkCbz, const Operand &opnd, bool is64BitOnly) const { if (bb.GetKind() != BB::kBBIf) { return false; @@ -4596,10 +6126,16 @@ bool DeleteMovAfterCbzOrCbnzAArch64::PredBBCheck(BB &bb, bool checkCbz, const Op return false; } MOperator mOp = condBr->GetMachineOpcode(); - if (checkCbz && mOp != MOP_wcbz && mOp != MOP_xcbz) { + if (is64BitOnly && checkCbz && mOp != MOP_xcbz) { return false; } - if (!checkCbz && mOp != MOP_xcbnz && mOp != MOP_wcbnz) { + if (is64BitOnly && !checkCbz && mOp != MOP_xcbnz) { + return false; + } + if (!is64BitOnly && checkCbz && mOp != MOP_xcbz && mOp != MOP_wcbz) { + return false; + } + if (!is64BitOnly && !checkCbz && mOp != MOP_xcbnz && mOp != MOP_wcbnz) { return false; } return RegOperand::IsSameRegNO(condBr->GetOperand(kInsnFirstOpnd), opnd); @@ -4636,7 +6172,7 @@ bool DeleteMovAfterCbzOrCbnzAArch64::OpndDefByMovZero(const Insn &insn) const bool DeleteMovAfterCbzOrCbnzAArch64::NoPreDefine(Insn &testInsn) const { Insn *nextInsn = nullptr; - for (Insn *insn = testInsn.GetBB()->GetFirstInsn(); insn != nullptr && insn != &testInsn; insn = nextInsn) { + for (Insn *insn = testInsn.GetBB()->GetFirstMachineInsn(); insn != nullptr && insn != &testInsn; insn = nextInsn) { nextInsn = insn->GetNextMachineInsn(); if (!insn->IsMachineInstruction()) { continue; @@ -4660,7 +6196,7 @@ bool DeleteMovAfterCbzOrCbnzAArch64::NoPreDefine(Insn &testInsn) const return false; } } else if (opnd.IsList()) { - for (auto operand : static_cast(opnd).GetOperands()) { + for (auto &operand : static_cast(opnd).GetOperands()) { if (RegOperand::IsSameRegNO(testInsn.GetOperand(kInsnFirstOpnd), *operand)) { return false; } @@ -4674,9 +6210,64 @@ bool DeleteMovAfterCbzOrCbnzAArch64::NoPreDefine(Insn &testInsn) const } return true; } + +bool DeleteMovAfterCbzOrCbnzAArch64::NoMoreThan32BitUse(Insn &testInsn) const +{ + auto &testOpnd = static_cast(testInsn.GetOperand(kFirstOpnd)); + InsnSet regUseInsnSet = cgFunc.GetRD()->FindUseForRegOpnd(testInsn, kInsnFirstOpnd, false); + for (auto useInsn : regUseInsnSet) { + MOperator mop = useInsn->GetMachineOpcode(); + if (mop == MOP_pseudo_ret_int) { + if (cgFunc.GetFunction().GetReturnType()->GetSize() > k4ByteSize) { + return false; + } + continue; + } + uint32 optSize = useInsn->GetOperandSize(); + const InsnDesc *md = useInsn->GetDesc(); + for (uint32 i = 0; i < optSize; i++) { + auto &opnd = useInsn->GetOperand(i); + const auto *opndDesc = md->GetOpndDes(i); + if (opndDesc->IsDef()) { + continue; + } + if (opnd.IsRegister()) { + auto ®Opnd = static_cast(opnd); + if (RegOperand::IsSameRegNO(regOpnd, testOpnd) && opndDesc->GetSize() > k32BitSize) { + return false; + } + } else if (opnd.IsMemoryAccessOperand()) { + auto &memOpnd = static_cast(opnd); + auto *baseOpnd = memOpnd.GetBaseRegister(); + auto *indexOpnd = memOpnd.GetIndexRegister(); + if ((baseOpnd != nullptr) && (RegOperand::IsSameRegNO(*baseOpnd, testOpnd)) && + (baseOpnd->GetSize() > k32BitSize)) { + return false; + } + if ((indexOpnd != nullptr) && (RegOperand::IsSameRegNO(*indexOpnd, testOpnd)) && + (indexOpnd->GetSize() > k32BitSize)) { + return false; + } + } else if (opnd.IsList()) { + auto &listOpnd = static_cast(opnd); + for (auto *regOpnd : std::as_const(listOpnd.GetOperands())) { + if (RegOperand::IsSameRegNO(*regOpnd, testOpnd) && regOpnd->GetSize() > k32BitSize) { + return false; + } + } + } + } + } + return true; +} + void DeleteMovAfterCbzOrCbnzAArch64::ProcessBBHandle(BB *processBB, const BB &bb, const Insn &insn) const { - FOR_BB_INSNS_SAFE(processInsn, processBB, nextProcessInsn) { + DEBUG_ASSERT(processBB != nullptr, "process_bb is null in ProcessBBHandle"); + MOperator condBrMop = insn.GetMachineOpcode(); + bool is64BitOnly = (condBrMop == MOP_xcbz || condBrMop == MOP_xcbnz); + FOR_BB_INSNS_SAFE(processInsn, processBB, nextProcessInsn) + { nextProcessInsn = processInsn->GetNextMachineInsn(); if (!processInsn->IsMachineInstruction()) { continue; @@ -4698,7 +6289,7 @@ void DeleteMovAfterCbzOrCbnzAArch64::ProcessBBHandle(BB *processBB, const BB &bb if (processBBPred == &bb) { continue; } - if (!PredBBCheck(*processBBPred, true, processInsn->GetOperand(kInsnFirstOpnd))) { + if (!PredBBCheck(*processBBPred, true, processInsn->GetOperand(kInsnFirstOpnd), is64BitOnly)) { toDoOpt = false; break; } @@ -4711,12 +6302,15 @@ void DeleteMovAfterCbzOrCbnzAArch64::ProcessBBHandle(BB *processBB, const BB &bb } /* for cbnz pred, there is one at most */ if (!PredBBCheck(*processBBPred, processBBPred != processBB->GetPrev(), - processInsn->GetOperand(kInsnFirstOpnd))) { + processInsn->GetOperand(kInsnFirstOpnd), is64BitOnly)) { toDoOpt = false; break; } } } + if (!is64BitOnly && !NoMoreThan32BitUse(*processInsn)) { + toDoOpt = false; + } if (!toDoOpt) { continue; } @@ -4724,6 +6318,33 @@ void DeleteMovAfterCbzOrCbnzAArch64::ProcessBBHandle(BB *processBB, const BB &bb } } +void DeleteMovAfterCbzOrCbnzAArch64::Run(BB &bb, Insn &insn) +{ + if (bb.GetKind() != BB::kBBIf) { + return; + } + if (&insn != cgcfg->FindLastCondBrInsn(bb)) { + return; + } + if (!cgcfg->IsCompareAndBranchInsn(insn)) { + return; + } + BB *processBB = nullptr; + if (bb.GetNext() == maplebe::CGCFG::GetTargetSuc(bb)) { + return; + } + + MOperator condBrMop = insn.GetMachineOpcode(); + if (condBrMop == MOP_wcbnz || condBrMop == MOP_xcbnz) { + processBB = bb.GetNext(); + } else { + processBB = maplebe::CGCFG::GetTargetSuc(bb); + } + + DEBUG_ASSERT(processBB != nullptr, "process_bb is null in DeleteMovAfterCbzOrCbnzAArch64::Run"); + ProcessBBHandle(processBB, bb, insn); +} + /* ldr wn, [x1, wn, SXTW] * add x2, wn, x2 */ @@ -4778,31 +6399,143 @@ void ComplexMemOperandAddAArch64::Run(BB &bb, Insn &insn) } } -void DeleteMovAfterCbzOrCbnzAArch64::Run(BB &bb, Insn &insn) +Insn *CombineMovInsnBeforeCSelPattern::FindPrevMovInsn(const Insn &insn, regno_t regNo) const { - if (bb.GetKind() != BB::kBBIf) { - return; - } - if (&insn != cgcfg->FindLastCondBrInsn(bb)) { - return; - } - if (!cgcfg->IsCompareAndBranchInsn(insn)) { - return; - } - BB *processBB = nullptr; - if (bb.GetNext() == maplebe::CGCFG::GetTargetSuc(bb)) { - return; + for (Insn *curInsn = insn.GetPreviousMachineInsn(); curInsn != nullptr; + curInsn = curInsn->GetPreviousMachineInsn()) { + MOperator mop = curInsn->GetMachineOpcode(); + if ((mop == MOP_wmovri32 || mop == MOP_xmovri64) && + static_cast(curInsn->GetOperand(kInsnFirstOpnd)).GetRegisterNumber() == regNo) { + return curInsn; + } + // If the register is redefined between the mov and csel insns, the optimization cannot be performed. + if (curInsn->IsRegDefined(regNo)) { + break; + } } + return nullptr; +} - MOperator condBrMop = insn.GetMachineOpcode(); - if (condBrMop == MOP_wcbnz || condBrMop == MOP_xcbnz) { - processBB = bb.GetNext(); +Insn *CombineMovInsnBeforeCSelPattern::FindPrevCmpInsn(const Insn &insn) const +{ + for (Insn *curInsn = insn.GetPreviousMachineInsn(); curInsn != nullptr; + curInsn = curInsn->GetPreviousMachineInsn()) { + MOperator mop = curInsn->GetMachineOpcode(); + if (mop == MOP_wcmpri || mop == MOP_xcmpri) { + return curInsn; + } + } + return nullptr; +} + +bool CombineMovInsnBeforeCSelPattern::CheckCondition(Insn &insn) +{ + MOperator curMop = insn.GetMachineOpcode(); + if (curMop != MOP_wcselrrrc && curMop != MOP_xcselrrrc) { + return false; + } + + auto &condOpnd = static_cast(insn.GetOperand(kInsnFourthOpnd)); + if (condOpnd.GetCode() != CC_NE && condOpnd.GetCode() != CC_EQ) { + return false; + } + + auto &opnd1 = static_cast(insn.GetOperand(kInsnSecondOpnd)); + auto &opnd2 = static_cast(insn.GetOperand(kInsnThirdOpnd)); + regno_t regNo1 = opnd1.GetRegisterNumber(); + regno_t regNo2 = opnd2.GetRegisterNumber(); + if (regNo1 == regNo2) { + return false; + } + + insnMov1 = FindPrevMovInsn(insn, regNo1); + if (insnMov1 == nullptr) { + return false; + } + insnMov2 = FindPrevMovInsn(insn, regNo2); + if (insnMov2 == nullptr) { + return false; + } + cmpInsn = FindPrevCmpInsn(insn); + if (cmpInsn == nullptr) { + return false; + } + + auto &cmpImmOpnd = static_cast(cmpInsn->GetOperand(kInsnThirdOpnd)); + auto &immOpnd1 = static_cast(insnMov1->GetOperand(kInsnSecondOpnd)); + auto &immOpnd2 = static_cast(insnMov2->GetOperand(kInsnSecondOpnd)); + auto maxImm = std::max(immOpnd1.GetValue(), immOpnd2.GetValue()); + auto minImm = std::min(immOpnd1.GetValue(), immOpnd2.GetValue()); + // to avoid difference value of imm1 and imm2 overflows + if (minImm < 0 && maxImm >= minImm - INT64_MIN) { + return false; + } + auto diffValue = maxImm - minImm; + if (diffValue == 0 || cmpImmOpnd.GetValue() != diffValue) { + return false; + } + // condition 5 + if (immOpnd1.GetValue() < immOpnd2.GetValue() && condOpnd.GetCode() != CC_NE && diffValue != 1) { + return false; + } + // condition 6 + if (immOpnd1.GetValue() > immOpnd2.GetValue() && condOpnd.GetCode() != CC_EQ && diffValue != 1) { + return false; + } + if (immOpnd1.GetValue() < immOpnd2.GetValue()) { + needReverseCond = true; + } + if (diffValue == 1 && ((immOpnd1.GetValue() < immOpnd2.GetValue() && condOpnd.GetCode() != CC_NE) || + (immOpnd1.GetValue() > immOpnd2.GetValue() && condOpnd.GetCode() != CC_EQ))) { + needCsetInsn = true; + } + + if (IfOperandIsLiveAfterInsn(opnd1, insn) || IfOperandIsLiveAfterInsn(opnd2, insn)) { + return false; + } + + return true; +} + +void CombineMovInsnBeforeCSelPattern::Run(BB &bb, Insn &insn) +{ + if (!CheckCondition(insn)) { + return; + } + + uint32 opndSize = insn.GetDesc()->opndMD[kInsnFirstOpnd]->GetSize(); + MOperator mOp = opndSize <= k32BitSize ? MOP_waddrri12 : MOP_xaddrri12; + auto &opnd0 = insn.GetOperand(kInsnFirstOpnd); + auto &cmpSrcopnd = cmpInsn->GetOperand(kInsnSecondOpnd); + auto *aarFunc = static_cast(cgFunc); + auto *condOpnd = static_cast(&insn.GetOperand(kInsnFourthOpnd)); + CondOperand &reverseCondOpnd = aarFunc->GetCondOperand(GetReverseCC(condOpnd->GetCode())); + if (needReverseCond) { + condOpnd = &reverseCondOpnd; + } + + // csel insn or cset insn + Insn *newInsn = nullptr; + // cset insn + if (needCsetInsn) { + MOperator newMop = opndSize <= k32BitSize ? MOP_wcsetrc : MOP_xcsetrc; + newInsn = &cgFunc->GetInsnBuilder()->BuildInsn(newMop, opnd0, *condOpnd, insn.GetOperand(kInsnFifthOpnd)); } else { - processBB = maplebe::CGCFG::GetTargetSuc(bb); + // csel insn + newInsn = &cgFunc->GetInsnBuilder()->BuildInsn(insn.GetMachineOpcode(), opnd0, cmpSrcopnd, + cgFunc->GetZeroOpnd(opndSize), *condOpnd, + insn.GetOperand(kInsnFifthOpnd)); } - DEBUG_ASSERT(processBB != nullptr, "process_bb is null in DeleteMovAfterCbzOrCbnzAArch64::Run"); - ProcessBBHandle(processBB, bb, insn); + auto &immOpnd1 = static_cast(insnMov1->GetOperand(kInsnSecondOpnd)); + auto &immOpnd2 = static_cast(insnMov2->GetOperand(kInsnSecondOpnd)); + int64 value = immOpnd1.GetValue() > immOpnd2.GetValue() ? immOpnd2.GetValue() : immOpnd1.GetValue(); + // add Insn + auto &newImmOpnd = aarFunc->CreateImmOperand(value, k12BitSize, false); + Insn &addInsn = cgFunc->GetInsnBuilder()->BuildInsn(mOp, opnd0, opnd0, newImmOpnd); + + bb.InsertInsnAfter(insn, addInsn); + bb.ReplaceInsn(insn, *newInsn); } MOperator OneHoleBranchesPreAArch64::FindNewMop(const BB &bb, const Insn &insn) const @@ -5124,6 +6857,135 @@ void RemoveIncRefPattern::Run(BB &bb, Insn &insn) bb.RemoveInsn(*insnMov1); } +bool LdrStrRevPattern::IsAdjacentMem(const MemOperand &memOperandLow, const MemOperand &memOperandHigh) const +{ + OfstOperand *highOfstOpnd = memOperandHigh.GetOffsetImmediate(); + int64 highOfstVal = highOfstOpnd ? highOfstOpnd->GetOffsetValue() : 0; + OfstOperand *lowOfstOpnd = memOperandLow.GetOffsetImmediate(); + int64 lowOfstVal = lowOfstOpnd ? lowOfstOpnd->GetOffsetValue() : 0; + if (highOfstVal - lowOfstVal != k8BitSize / kBitsPerByte) { + return false; + } + if (!RegOperand::IsSameReg(*memOperandLow.GetBaseRegister(), *memOperandHigh.GetBaseRegister())) { + return false; + } + return true; +} + +void LdrStrRevPattern::Run(BB &bb, Insn &insn) +{ + AArch64CGFunc *aarch64CGFunc = static_cast(cgFunc); + if (!CheckCondition(insn)) { + return; + } + Insn *newInsn = nullptr; + Insn *revInsn = nullptr; + Insn *lastInsn = nullptr; + MOperator newMemMop = insn.GetMachineOpcode() == MOP_wstrb ? MOP_wstrh : MOP_wldrh; + MemOperand *newMemOperand = + aarch64CGFunc->CreateMemOperand(MemOperand::kAddrModeBOi, k8BitSize * 2, *adjacentMemOpnd->GetBaseRegister(), + nullptr, adjacentMemOpnd->GetOffsetOperand(), nullptr); + if (isStrInsn) { + newInsn = &cgFunc->GetInsnBuilder()->BuildInsn(newMemMop, lsrInsn->GetOperand(kFirstOpnd), *newMemOperand); + revInsn = &cgFunc->GetInsnBuilder()->BuildInsn(MOP_wrevrr16, lsrInsn->GetOperand(kFirstOpnd), + lsrInsn->GetOperand(kSecondOpnd)); + } else { + newInsn = &cgFunc->GetInsnBuilder()->BuildInsn(newMemMop, insn.GetOperand(kFirstOpnd), *newMemOperand); + revInsn = &cgFunc->GetInsnBuilder()->BuildInsn(MOP_wrevrr16, lsrInsn->GetOperand(kFirstOpnd), + insn.GetOperand(kFirstOpnd)); + } + if (isStrInsn) { + bb.InsertInsnAfter(insn, *newInsn); + bb.InsertInsnAfter(insn, *revInsn); + lastInsn = revInsn; + } else { + bb.InsertInsnAfter(insn, *revInsn); + bb.InsertInsnAfter(insn, *newInsn); + lastInsn = newInsn; + } + bb.RemoveInsn(insn); + bb.RemoveInsn(*lsrInsn); + bb.RemoveInsn(*adjacentInsn); + SetCurrInsn(lastInsn); + return; +} + +bool LdrStrRevPattern::CheckCondition(Insn &insn) +{ + MOperator mop = insn.GetMachineOpcode(); + isStrInsn = mop == MOP_wstrb; + curMemOpnd = static_cast(insn.GetMemOpnd()); + if (curMemOpnd->GetAddrMode() != MemOperand::kAddrModeBOi) { + return false; + } + if (isStrInsn) { + adjacentInsn = insn.GetPreviousMachineInsn(); + if (adjacentInsn == nullptr || adjacentInsn->GetMachineOpcode() != mop) { + return false; + } + adjacentMemOpnd = static_cast(adjacentInsn->GetMemOpnd()); + lsrInsn = adjacentInsn->GetPreviousMachineInsn(); + if (lsrInsn == nullptr || + (lsrInsn->GetMachineOpcode() != MOP_xlsrrri6 && lsrInsn->GetMachineOpcode() != MOP_wlsrrri5)) { + return false; + } + RegOperand &lsrDst = static_cast(lsrInsn->GetOperand(kFirstOpnd)); + RegOperand &lsrSrc = static_cast(lsrInsn->GetOperand(kSecondOpnd)); + ImmOperand &lsrImm = static_cast(lsrInsn->GetOperand(kThirdOpnd)); + + RegOperand &currSrc = static_cast(insn.GetOperand(kFirstOpnd)); + RegOperand &adjacentSrc = static_cast(adjacentInsn->GetOperand(kFirstOpnd)); + if (lsrImm.GetValue() != k8BitSize) { + return false; + } + if (!RegOperand::IsSameReg(lsrDst, adjacentSrc) || !RegOperand::IsSameReg(lsrSrc, currSrc)) { + return false; + } + if (!IsAdjacentMem(*adjacentMemOpnd, *curMemOpnd)) { + return false; + } + if (IfOperandIsLiveAfterInsn(lsrDst, insn)) { + return false; + } + } else { + adjacentInsn = insn.GetNextMachineInsn(); + if (adjacentInsn == nullptr || adjacentInsn->GetMachineOpcode() != mop) { + return false; + } + adjacentMemOpnd = static_cast(adjacentInsn->GetMemOpnd()); + lsrInsn = adjacentInsn->GetNextMachineInsn(); + if (lsrInsn == nullptr || + (lsrInsn->GetMachineOpcode() != MOP_waddrrrs && lsrInsn->GetMachineOpcode() != MOP_xaddrrrs)) { + return false; + } + auto &addSrc1 = static_cast(lsrInsn->GetOperand(kSecondOpnd)); + auto &addSrc2 = static_cast(lsrInsn->GetOperand(kThirdOpnd)); + auto &shiftOpnd = static_cast(lsrInsn->GetOperand(kInsnFourthOpnd)); + + RegOperand &currSrc = static_cast(insn.GetOperand(kFirstOpnd)); + RegOperand &adjacentSrc = static_cast(adjacentInsn->GetOperand(kFirstOpnd)); + if (!RegOperand::IsSameReg(addSrc1, currSrc) || !RegOperand::IsSameReg(addSrc2, adjacentSrc)) { + return false; + } + if (shiftOpnd.GetShiftOp() != BitShiftOperand::kLSL || shiftOpnd.GetShiftAmount() != k8BitSize) { + return false; + } + if (!RegOperand::IsSameReg(*curMemOpnd->GetBaseRegister(), *adjacentMemOpnd->GetBaseRegister())) { + return false; + } + if (!IsAdjacentMem(*adjacentMemOpnd, *curMemOpnd)) { + return false; + } + if (IfOperandIsLiveAfterInsn(currSrc, *lsrInsn)) { + return false; + } + if (IfOperandIsLiveAfterInsn(adjacentSrc, *lsrInsn)) { + return false; + } + } + return true; +} + bool LongIntCompareWithZPattern::FindLondIntCmpWithZ(Insn &insn) { MOperator thisMop = insn.GetMachineOpcode(); @@ -5226,7 +7088,8 @@ void ComplexMemOperandAArch64::Run(BB &bb, Insn &insn) } MOperator nextMop = nextInsn->GetMachineOpcode(); - if (nextMop && ((nextMop >= MOP_wldrsb && nextMop <= MOP_dldp) || (nextMop >= MOP_wstrb && nextMop <= MOP_dstp))) { + if (nextMop > 0 && + ((nextMop >= MOP_wldrsb && nextMop <= MOP_dldp) || (nextMop >= MOP_wstrb && nextMop <= MOP_dstp))) { /* Check if base register of nextInsn and the dest operand of insn are identical. */ MemOperand *memOpnd = static_cast(nextInsn->GetMemOpnd()); DEBUG_ASSERT(memOpnd != nullptr, "memOpnd is null in AArch64Peep::ComplexMemOperandAArch64"); @@ -5277,6 +7140,13 @@ void ComplexMemOperandAArch64::Run(BB &bb, Insn &insn) return; } + auto &newBaseOpnd = static_cast(insn.GetOperand(kInsnSecondOpnd)); + MemOperand &newMemOpnd = aarch64CGFunc->GetOrCreateMemOpnd( + MemOperand::kAddrModeLo12Li, memOpnd->GetSize(), &newBaseOpnd, nullptr, &offOpnd, stImmOpnd.GetSymbol()); + if (!aarch64CGFunc->IsOperandImmValid(nextMop, &newMemOpnd, nextInsn->GetMemOpndIdx())) { + return; + } + if (cgFunc.GetMirModule().IsCModule()) { Insn *prevInsn = insn.GetPrev(); MOperator prevMop = prevInsn->GetMachineOpcode(); @@ -5287,9 +7157,6 @@ void ComplexMemOperandAArch64::Run(BB &bb, Insn &insn) prevStImmOpnd.SetOffset(offOpnd.GetValue()); } } - auto &newBaseOpnd = static_cast(insn.GetOperand(kInsnSecondOpnd)); - MemOperand &newMemOpnd = aarch64CGFunc->GetOrCreateMemOpnd( - MemOperand::kAddrModeLo12Li, memOpnd->GetSize(), &newBaseOpnd, nullptr, &offOpnd, stImmOpnd.GetSymbol()); nextInsn->SetMemOpnd(&newMemOpnd); bb.RemoveInsn(insn); @@ -5310,7 +7177,8 @@ void ComplexMemOperandPreAddAArch64::Run(BB &bb, Insn &insn) return; } MOperator nextMop = nextInsn->GetMachineOpcode(); - if (nextMop && ((nextMop >= MOP_wldrsb && nextMop <= MOP_dldr) || (nextMop >= MOP_wstrb && nextMop <= MOP_dstr))) { + if (nextMop > 0 && + ((nextMop >= MOP_wldrsb && nextMop <= MOP_dldr) || (nextMop >= MOP_wstrb && nextMop <= MOP_dstr))) { if (!IsMemOperandOptPattern(insn, *nextInsn)) { return; } @@ -5660,7 +7528,7 @@ void OneHoleBranchesAArch64::Run(BB &bb, Insn &insn) return; } auto &imm = static_cast(prevInsn->GetOperand(kInsnThirdOpnd)); - int n = logValueAtBase2(imm.GetValue()); + int n = LogValueAtBase2(imm.GetValue()); if (n < 0) { return; } @@ -5778,7 +7646,7 @@ void AndCmpBranchesToTbzAArch64::Run(BB &bb, Insn &insn) auto &immAnd = static_cast(prevPrevInsn->GetOperand(opndIdx)); auto &immCmp = static_cast(prevInsn->GetOperand(opndIdx)); if (immCmp.GetValue() == 0) { - int n = logValueAtBase2(immAnd.GetValue()); + int n = LogValueAtBase2(immAnd.GetValue()); if (n < 0) { return; } @@ -5815,8 +7683,8 @@ void AndCmpBranchesToTbzAArch64::Run(BB &bb, Insn &insn) bb.RemoveInsn(*prevInsn); bb.RemoveInsn(*prevPrevInsn); } else { - int n = logValueAtBase2(immAnd.GetValue()); - int m = logValueAtBase2(immCmp.GetValue()); + int n = LogValueAtBase2(immAnd.GetValue()); + int m = LogValueAtBase2(immCmp.GetValue()); if (n < 0 || m < 0 || n != m) { return; } @@ -5885,6 +7753,16 @@ void RemoveSxtBeforeStrAArch64::Run(BB &bb, Insn &insn) bb.RemoveInsn(*prevInsn); } +bool UbfxToUxtwPattern::CheckCondition(Insn &insn) +{ + ImmOperand &imm0 = static_cast(insn.GetOperand(kInsnThirdOpnd)); + ImmOperand &imm1 = static_cast(insn.GetOperand(kInsnFourthOpnd)); + if ((imm0.GetValue() != 0) || (imm1.GetValue() != k32BitSize)) { + return false; + } + return true; +} + void UbfxToUxtwPattern::Run(BB &bb, Insn &insn) { if (!CheckCondition(insn)) { @@ -5893,6 +7771,8 @@ void UbfxToUxtwPattern::Run(BB &bb, Insn &insn) Insn *newInsn = &cgFunc->GetInsnBuilder()->BuildInsn(MOP_xuxtw64, insn.GetOperand(kInsnFirstOpnd), insn.GetOperand(kInsnSecondOpnd)); bb.ReplaceInsn(insn, *newInsn); + SetCurrInsn(newInsn); + optSuccess = true; if (CG_PEEP_DUMP) { std::vector prevs; prevs.emplace_back(&insn); @@ -5900,16 +7780,339 @@ void UbfxToUxtwPattern::Run(BB &bb, Insn &insn) } } -bool UbfxToUxtwPattern::CheckCondition(Insn &insn) +bool NormRevTbzToTbzPattern::CheckCondition(Insn &insn) { - ImmOperand &imm0 = static_cast(insn.GetOperand(kInsnThirdOpnd)); - ImmOperand &imm1 = static_cast(insn.GetOperand(kInsnFourthOpnd)); - if ((imm0.GetValue() != 0) || (imm1.GetValue() != k32BitSize)) { + auto &revReg = static_cast(insn.GetOperand(kInsnFirstOpnd)); + for (Insn *nextInsn = insn.GetNextMachineInsn(); nextInsn != nullptr; nextInsn = nextInsn->GetNextMachineInsn()) { + MOperator useMop = nextInsn->GetMachineOpcode(); + auto &useReg = static_cast(nextInsn->GetOperand(kInsnFirstOpnd)); + if ((useMop == MOP_wtbnz || useMop == MOP_xtbnz || useMop == MOP_wtbz || useMop == MOP_xtbz) && + useReg.Equals(revReg)) { + if (IfOperandIsLiveAfterInsn(useReg, *nextInsn)) { + return false; + } + tbzInsn = nextInsn; + return true; + } + uint32 opndSize = nextInsn->GetOperandSize(); + for (uint32 i = 0; i < opndSize; i++) { + auto &duOpnd = nextInsn->GetOperand(i); + if (!duOpnd.IsRegister()) { + continue; + } + if ((static_cast(duOpnd)).GetRegisterNumber() != revReg.GetRegisterNumber()) { + continue; + } + return false; + } + } + return false; +} + +void NormRevTbzToTbzPattern::SetRev16Value(const uint32 &oldValue, uint32 &revValue) const +{ + switch (oldValue / k8BitSize) { + case k0BitSize: + case k2BitSize: + case k4BitSize: + case k6BitSize: + revValue = oldValue + k8BitSize; + break; + case k1BitSize: + case k3BitSize: + case k5BitSize: + case k7BitSize: + revValue = oldValue - k8BitSize; + break; + default: + CHECK_FATAL(false, "revValue must be the above value"); + } +} + +void NormRevTbzToTbzPattern::SetWrevValue(const uint32 &oldValue, uint32 &revValue) const +{ + switch (oldValue / k8BitSize) { + case k0BitSize: { + revValue = oldValue + k24BitSize; + break; + } + case k1BitSize: { + revValue = oldValue + k8BitSize; + break; + } + case k2BitSize: { + revValue = oldValue - k8BitSize; + break; + } + case k4BitSize: { + revValue = oldValue - k24BitSize; + break; + } + default: + CHECK_FATAL(false, "revValue must be the above value"); + } +} + +void NormRevTbzToTbzPattern::SetXrevValue(const uint32 &oldValue, uint32 &revValue) const +{ + switch (oldValue / k8BitSize) { + case k0BitSize: + revValue = oldValue + k56BitSize; + break; + case k1BitSize: + revValue = oldValue + k40BitSize; + break; + case k2BitSize: + revValue = oldValue + k24BitSize; + break; + case k3BitSize: + revValue = oldValue + k8BitSize; + break; + case k4BitSize: + revValue = oldValue - k8BitSize; + break; + case k5BitSize: + revValue = oldValue - k24BitSize; + break; + case k6BitSize: + revValue = oldValue - k40BitSize; + break; + case k7BitSize: + revValue = oldValue - k56BitSize; + break; + default: + CHECK_FATAL(false, "revValue must be the above value"); + } +} + +void NormRevTbzToTbzPattern::Run(BB &bb, Insn &insn) +{ + if (!CheckCondition(insn)) { + return; + } + auto &oldImmOpnd1 = static_cast(tbzInsn->GetOperand(kInsnSecondOpnd)); + uint32 oldValue = static_cast(oldImmOpnd1.GetValue()); + uint32 revValue = k0BitSize; + MOperator curMop = insn.GetMachineOpcode(); + if (curMop == MOP_wrevrr16) { + SetRev16Value(oldValue, revValue); + } else if (curMop == MOP_wrevrr) { + SetWrevValue(oldValue, revValue); + } else if (curMop == MOP_xrevrr) { + SetXrevValue(oldValue, revValue); + } + auto *aarFunc = static_cast(cgFunc); + ImmOperand &newImmOpnd = aarFunc->CreateImmOperand(revValue, k6BitSize, false); + MOperator useMop = tbzInsn->GetMachineOpcode(); + Insn &newInsn = cgFunc->GetInsnBuilder()->BuildInsn(useMop, insn.GetOperand(kInsnSecondOpnd), newImmOpnd, + tbzInsn->GetOperand(kInsnThirdOpnd)); + bb.ReplaceInsn(*tbzInsn, newInsn); + optSuccess = true; + /* dump pattern info */ + if (CG_PEEP_DUMP) { + std::vector prevs; + (void)prevs.emplace_back(&insn); + DumpAfterPattern(prevs, tbzInsn, &newInsn); + } +} + +Insn *AddSubMergeLdStPattern::FindRegInBB(const Insn &insn, bool isAbove) const +{ + regno_t regNO = static_cast(insn.GetOperand(kInsnFirstOpnd)).GetRegisterNumber(); + for (Insn *resInsn = isAbove ? insn.GetPreviousMachineInsn() : insn.GetNextMachineInsn(); resInsn != nullptr; + resInsn = isAbove ? resInsn->GetPreviousMachineInsn() : resInsn->GetNextMachineInsn()) { + if (resInsn->GetDesc()->IsCall() || resInsn->GetDesc()->IsInlineAsm() || + resInsn->GetDesc()->IsSpecialIntrinsic()) { + return nullptr; + } + if (resInsn->ScanReg(regNO)) { + return resInsn; + } + } + return nullptr; +} + +bool AddSubMergeLdStPattern::CheckCondition(Insn &insn) +{ + insnDefReg = &static_cast(insn.GetOperand(kInsnFirstOpnd)); + insnUseReg = &static_cast(insn.GetOperand(kInsnSecondOpnd)); + regno_t insnDefRegNO = insnDefReg->GetRegisterNumber(); + regno_t insnUseRegNO = insnUseReg->GetRegisterNumber(); + if (insnDefRegNO != insnUseRegNO) { + return false; + } + // Do not combine x16 until cgaggressiveopt + if (insnDefReg->IsPhysicalRegister() && insnDefRegNO == R16) { + return false; + } + nextInsn = FindRegInBB(insn, false); + prevInsn = FindRegInBB(insn, true); + isAddSubFront = CheckIfCanBeMerged(nextInsn, insn); + isLdStFront = CheckIfCanBeMerged(prevInsn, insn); + // If prev & next all can be merged, only one will be merged, otherwise #imm will be add/sub twice. + if (isAddSubFront && isLdStFront) { + isLdStFront = false; + } + return isAddSubFront || isLdStFront; +} + +bool AddSubMergeLdStPattern::CheckIfCanBeMerged(const Insn *adjacentInsn, const Insn & /* insn */) +{ + if (adjacentInsn == nullptr || adjacentInsn->IsVectorOp() || (!adjacentInsn->AccessMem())) { + return false; + } + Operand &opnd = adjacentInsn->IsLoadStorePair() ? adjacentInsn->GetOperand(kInsnThirdOpnd) + : adjacentInsn->GetOperand(kInsnSecondOpnd); + if (opnd.GetKind() != Operand::kOpdMem) { + return false; + } + MemOperand *memOpnd = &static_cast(opnd); + // load/store memopnd offset value must be #0 + if (memOpnd->GetAddrMode() != MemOperand::kAddrModeBOi || AArch64isa::GetMemOpndOffsetValue(memOpnd) != 0) { + return false; + } + RegOperand *memUseReg = memOpnd->GetBaseRegister(); + regno_t insnDefRegNO = insnDefReg->GetRegisterNumber(); + regno_t memUseRegNO = memUseReg->GetRegisterNumber(); + if (insnDefRegNO != memUseRegNO) { + return false; + } + // When load/store insn def & use regno are the same, it will trigger unpredictable transfer with writeback. + regno_t ldstDefRegNO0 = static_cast(adjacentInsn->GetOperand(kInsnFirstOpnd)).GetRegisterNumber(); + if (ldstDefRegNO0 == memUseRegNO) { return false; } + if (adjacentInsn->IsLoadStorePair()) { + regno_t ldstDefRegNO1 = + static_cast(adjacentInsn->GetOperand(kInsnSecondOpnd)).GetRegisterNumber(); + if (ldstDefRegNO1 == memUseRegNO) { + return false; + } + } return true; } +void AddSubMergeLdStPattern::Run(BB &bb, Insn &insn) +{ + if (!CheckCondition(insn)) { + return; + } + insnToBeReplaced = isAddSubFront ? nextInsn : prevInsn; + // isInsnAdd returns true -- add, isInsnAdd returns false -- sub. + isInsnAdd = (insn.GetMachineOpcode() == MOP_xaddrri12); + int64 immVal = static_cast(insn.GetOperand(kInsnThirdOpnd)).GetValue(); + // Pre/Post-index simm cannot be absent, when ofstVal is #0, the assembly file will appear memopnd: [x0]! + if (immVal == static_cast(k0BitSize)) { + return; + } + Operand &opnd = insnToBeReplaced->IsLoadStorePair() ? insnToBeReplaced->GetOperand(kInsnThirdOpnd) + : insnToBeReplaced->GetOperand(kInsnSecondOpnd); + MemOperand *memOpnd = &static_cast(opnd); + ImmOperand &newImmOpnd = + static_cast(cgFunc)->CreateImmOperand((isInsnAdd ? immVal : (-immVal)), k64BitSize, true); + MemOperand *newMemOpnd = static_cast(cgFunc)->CreateMemOperand( + MemOperand::kAddrModeBOi, memOpnd->GetSize(), *insnUseReg, nullptr, &newImmOpnd, nullptr); + Insn *newInsn = nullptr; + if (insnToBeReplaced->IsLoadStorePair()) { + newInsn = &static_cast(cgFunc)->GetInsnBuilder()->BuildInsn( + insnToBeReplaced->GetMachineOpcode(), insnToBeReplaced->GetOperand(kInsnFirstOpnd), + insnToBeReplaced->GetOperand(kInsnSecondOpnd), *newMemOpnd); + } else { + newInsn = &static_cast(cgFunc)->GetInsnBuilder()->BuildInsn( + insnToBeReplaced->GetMachineOpcode(), insnToBeReplaced->GetOperand(kInsnFirstOpnd), *newMemOpnd); + } + // Both [RSP, #imm]! and [RSP], #imm should be set true for stackdef. + if (insnUseReg->GetRegisterNumber() == RSP) { + newInsn->SetStackDef(true); + } + bb.ReplaceInsn(*insnToBeReplaced, *newInsn); + bb.RemoveInsn(insn); +} + +void UbfxAndMergetPattern::Run(BB &bb, Insn &insn) +{ + if (!CheckCondition(insn)) { + return; + } + int64 newLsb = currLsb + prevLsb; + int64 newWidth = std::min(prevWidth - currLsb, currWidth); + if (newLsb + newWidth > k32BitSize) { + newMop = MOP_xubfxrri6i6; + } + CHECK_FATAL(newLsb >= 0, "must be"); + CHECK_FATAL(newLsb + newWidth <= k64BitSize, "must be"); + Insn *newInsn = nullptr; + if (newWidth <= 0) { + // two insns have no overlap + ImmOperand &zeroOpnd = static_cast(cgFunc)->CreateImmOperand(0, k64BitSize, true); + newInsn = &static_cast(cgFunc)->GetInsnBuilder()->BuildInsn( + MOP_xmovri64, insn.GetOperand(kInsnFirstOpnd), zeroOpnd); + } else { + ImmOperand &newLsbImmOpnd = static_cast(cgFunc)->CreateImmOperand(newLsb, k64BitSize, true); + ImmOperand &newWidthImmOpnd = + static_cast(cgFunc)->CreateImmOperand(newWidth, k64BitSize, true); + newInsn = &static_cast(cgFunc)->GetInsnBuilder()->BuildInsn( + newMop, insn.GetOperand(kInsnFirstOpnd), *prevSrc, newLsbImmOpnd, newWidthImmOpnd); + } + bb.ReplaceInsn(insn, *newInsn); + if (ssaInfo) { + ssaInfo->ReplaceInsn(insn, *newInsn); + } + return; +} + +bool UbfxAndMergetPattern::CheckCondition(Insn &insn) +{ + // and def src imm + // ubfx def src imm imm + auto &srcReg = static_cast(insn.GetOperand(kInsnSecondOpnd)); + Insn *prevInsn = ssaInfo->GetDefInsn(srcReg); + // not in ssa form + if (prevInsn == nullptr) { + return false; + } + if (prevInsn->GetMachineOpcode() != MOP_xubfxrri6i6 && prevInsn->GetMachineOpcode() != MOP_wubfxrri5i5) { + return false; + } + auto &prevLsbOperand = static_cast(prevInsn->GetOperand(kInsnThirdOpnd)); + auto &prevWidthOperand = static_cast(prevInsn->GetOperand(kInsnFourthOpnd)); + prevSrc = &static_cast(prevInsn->GetOperand(kInsnSecondOpnd)); + prevLsb = prevLsbOperand.GetValue(); + prevWidth = prevWidthOperand.GetValue(); + // do not prop phaysical reg， it may cross call. + if (!prevSrc->IsSSAForm() || prevSrc->IsPhysicalRegister()) { + return false; + } + if (insn.GetMachineOpcode() == MOP_xubfxrri6i6 || insn.GetMachineOpcode() == MOP_wubfxrri5i5) { + auto &currLsbOperand = static_cast(insn.GetOperand(kInsnThirdOpnd)); + auto &currWidthOperand = static_cast(insn.GetOperand(kInsnFourthOpnd)); + currLsb = currLsbOperand.GetValue(); + currWidth = currWidthOperand.GetValue(); + newMop = insn.GetMachineOpcode(); + } else if (insn.GetMachineOpcode() == MOP_xandrri13 || insn.GetMachineOpcode() == MOP_wandrri12) { + // and R1 R0 0xFF = ubfx R1 R0 0 8 + auto &andImm = static_cast(insn.GetOperand(kInsnThirdOpnd)); + int64 andVal = andImm.GetValue(); + if (!IsAllOneToMSB(andVal)) { + return false; + } + currLsb = 0; + currWidth = GetMSB(andVal); + newMop = insn.GetMachineOpcode() == MOP_xandrri13 ? MOP_xubfxrri6i6 : MOP_wubfxrri5i5; + } + return true; +} + +bool UbfxAndMergetPattern::IsAllOneToMSB(int64 val) const +{ + return ((static_cast(val) + 1) & static_cast(val)) == 0; +} + +int32 UbfxAndMergetPattern::GetMSB(int64 val) const +{ + return static_cast(k64BitSize - static_cast(__builtin_clzll(static_cast(val)))); +} + void UbfxAndCbzToTbzPattern::Run(BB &bb, Insn &insn) { Operand &opnd2 = static_cast(insn.GetOperand(kInsnSecondOpnd)); @@ -5921,16 +8124,12 @@ void UbfxAndCbzToTbzPattern::Run(BB &bb, Insn &insn) MOperator nextMop = useInsn->GetMachineOpcode(); switch (nextMop) { case MOP_wcbz: - newMop = MOP_wtbz; - break; case MOP_xcbz: - newMop = MOP_xtbz; + newMop = opnd2.GetSize() == k64BitSize ? MOP_xtbz : MOP_wtbz; break; case MOP_wcbnz: - newMop = MOP_wtbnz; - break; case MOP_xcbnz: - newMop = MOP_xtbnz; + newMop = opnd2.GetSize() == k64BitSize ? MOP_xtbnz : MOP_wtbnz; break; default: return; @@ -5941,8 +8140,12 @@ void UbfxAndCbzToTbzPattern::Run(BB &bb, Insn &insn) Insn *newInsn = &cgFunc->GetInsnBuilder()->BuildInsn(newMop, opnd2, imm3, label); BB *useInsnBB = useInsn->GetBB(); useInsnBB->ReplaceInsn(*useInsn, *newInsn); - /* update ssa info */ - ssaInfo->ReplaceInsn(*useInsn, *newInsn); + if (ssaInfo) { + // update ssa info + ssaInfo->ReplaceInsn(*useInsn, *newInsn); + } else { + useInsnBB->RemoveInsn(insn); + } optSuccess = true; if (CG_PEEP_DUMP) { std::vector prevs; @@ -5956,13 +8159,26 @@ bool UbfxAndCbzToTbzPattern::CheckCondition(Insn &insn) ImmOperand &imm4 = static_cast(insn.GetOperand(kInsnFourthOpnd)); RegOperand &opnd1 = static_cast(insn.GetOperand(kInsnFirstOpnd)); InsnSet useInsns = GetAllUseInsn(opnd1); - if (useInsns.size() != 1) { - return false; + if (ssaInfo) { + InsnSet useInsns = GetAllUseInsn(opnd1); + if (useInsns.size() != 1) { + return false; + } + useInsn = *useInsns.begin(); + } else { + useInsn = insn.GetNextMachineInsn(); } - useInsn = *useInsns.begin(); if (useInsn == nullptr) { return false; } + if (!ssaInfo) { + regno_t regNO1 = static_cast(insn.GetOperand(kInsnFirstOpnd)).GetRegisterNumber(); + regno_t regNO2 = static_cast(useInsn->GetOperand(kInsnFirstOpnd)).GetRegisterNumber(); + if ((regNO1 != regNO2) || + IfOperandIsLiveAfterInsn(static_cast(insn.GetOperand(kInsnFirstOpnd)), *useInsn)) { + return false; + } + } if (imm4.GetValue() == 1) { switch (useInsn->GetMachineOpcode()) { case MOP_wcbz: @@ -5977,6 +8193,194 @@ bool UbfxAndCbzToTbzPattern::CheckCondition(Insn &insn) return false; } +bool AddCmpZeroPattern::CheckAddCmpZeroCheckAdd(const Insn &insn) const +{ + MOperator mop = prevInsn->GetMachineOpcode(); + switch (mop) { + case MOP_xaddrrr: + case MOP_waddrrr: + case MOP_xaddrrrs: + case MOP_waddrrrs: { + RegOperand opnd0 = static_cast(prevInsn->GetOperand(kInsnFirstOpnd)); + RegOperand opnd = static_cast(insn.GetOperand(kInsnSecondOpnd)); + if (opnd0.Equals(opnd) && insn.GetDesc()->GetOpndDes(kInsnSecondOpnd)->GetSize() == + prevInsn->GetDesc()->GetOpndDes(kInsnFirstOpnd)->GetSize()) { + return true; + } else { + return false; + } + } + case MOP_waddrri12: + case MOP_xaddrri12: { + RegOperand opnd0 = static_cast(prevInsn->GetOperand(kInsnFirstOpnd)); + RegOperand opnd = static_cast(insn.GetOperand(kInsnSecondOpnd)); + if (!(opnd0.Equals(opnd) && insn.GetDesc()->GetOpndDes(kInsnSecondOpnd)->GetSize() == + prevInsn->GetDesc()->GetOpndDes(kInsnFirstOpnd)->GetSize())) { + return false; + } + auto &immOpnd = static_cast(prevInsn->GetOperand(kInsnThirdOpnd)); + auto *aarch64CGFunc = static_cast(cgFunc); + if (aarch64CGFunc->IsOperandImmValid(prevInsn->GetMachineOpcode(), &immOpnd, kInsnThirdOpnd)) { + return true; + } else { + return false; + } + } + default: + break; + } + return false; +} + +bool AddCmpZeroPattern::CheckAddCmpZeroContinue(const Insn &insn, const RegOperand &opnd) const +{ + // check if insn will redef target reg or status reg + if (insn.GetDesc()->IsCall() || insn.GetDesc()->IsSpecialCall()) { + return false; + } + for (uint32 i = 0; i < insn.GetOperandSize(); ++i) { + if (insn.GetDesc()->GetOpndDes(i) == &OpndDesc::CCS) { + return false; + } + if (insn.GetOperand(i).IsRegister()) { + RegOperand &opnd0 = static_cast(insn.GetOperand(i)); + if (insn.GetDesc()->GetOpndDes(i)->IsDef() && opnd0.RegNumEqual(opnd)) { + return false; + } + } + } + return true; +} + +bool AddCmpZeroPattern::CheckCondition(Insn &insn) +{ + auto &opnd2 = static_cast(insn.GetOperand(kInsnSecondOpnd)); + auto &opnd3 = static_cast(insn.GetOperand(kInsnThirdOpnd)); + if (!opnd3.IsZero()) { + return false; + } + prevInsn = insn.GetPrev(); + while (prevInsn != nullptr) { + if (!prevInsn->IsMachineInstruction()) { + prevInsn = prevInsn->GetPrev(); + continue; + } + if (CheckAddCmpZeroCheckAdd(insn)) { + if (CheckAddCmpZeroCheckCond(insn)) { + return (prevInsn != nullptr); + } else { + return false; + } + } + if (!CheckAddCmpZeroContinue(*prevInsn, opnd2)) { + return false; + } + prevInsn = prevInsn->GetPrev(); + } + return (prevInsn != nullptr); +} + +bool AddCmpZeroPattern::CheckAddCmpZeroCheckCond(const Insn &insn) const +{ + Insn *nextInsn = insn.GetNext(); + while (nextInsn != nullptr) { + if (!nextInsn->IsMachineInstruction()) { + nextInsn = nextInsn->GetNext(); + continue; + } + for (uint32 i = 0; i < nextInsn->GetOperandSize(); ++i) { + if (nextInsn->GetDesc()->GetOpndDes(i) == &OpndDesc::Cond) { + CondOperand &cond = static_cast(nextInsn->GetOperand(i)); + if (cond.GetCode() == CC_EQ) { + return true; + } else { + return false; + } + } + } + nextInsn = nextInsn->GetNext(); + } + return false; +} + +void AddCmpZeroPattern::Run(BB &bb, Insn &insn) +{ + MOperator mop = insn.GetMachineOpcode(); + if (mop != MOP_wcmpri && mop != MOP_xcmpri) { + return; + } + if (!CheckCondition(insn)) { + return; + } + + bool isAddShift = false; + MOperator newMop = GetMopUpdateAPSR(prevInsn->GetMachineOpcode(), isAddShift); + Insn *newInsn = nullptr; + if (isAddShift) { + newInsn = &cgFunc->GetInsnBuilder()->BuildInsn( + newMop, insn.GetOperand(kInsnFirstOpnd), prevInsn->GetOperand(kInsnFirstOpnd), + prevInsn->GetOperand(kInsnSecondOpnd), prevInsn->GetOperand(kInsnThirdOpnd), + prevInsn->GetOperand(kInsnFourthOpnd)); + } else { + newInsn = &cgFunc->GetInsnBuilder()->BuildInsn( + newMop, insn.GetOperand(kInsnFirstOpnd), prevInsn->GetOperand(kInsnFirstOpnd), + prevInsn->GetOperand(kInsnSecondOpnd), prevInsn->GetOperand(kInsnThirdOpnd)); + } + bb.ReplaceInsn(*prevInsn, *newInsn); + bb.RemoveInsn(insn); + optSuccess = true; + if (CG_PEEP_DUMP) { + std::vector prevs; + (void)prevs.emplace_back(prevInsn); + DumpAfterPattern(prevs, newInsn, nullptr); + } +} + +bool ComplexExtendWordLslPattern::CheckCondition(Insn &insn) +{ + if (insn.GetMachineOpcode() != MOP_xsxtw64 && insn.GetMachineOpcode() != MOP_xuxtw64) { + return false; + } + useInsn = insn.GetNextMachineInsn(); + if (useInsn == nullptr) { + return false; + } + MOperator nextMop = useInsn->GetMachineOpcode(); + if (nextMop != MOP_xlslrri6) { + return false; + } + return true; +} + +void ComplexExtendWordLslPattern::Run(BB &bb, Insn &insn) +{ + if (!CheckCondition(insn)) { + return; + } + MOperator curMop = insn.GetMachineOpcode(); + auto &lslImmOpnd = static_cast(useInsn->GetOperand(kInsnThirdOpnd)); + DEBUG_ASSERT(lslImmOpnd.GetValue() >= 0, "invalid immOpnd of lsl"); + if (lslImmOpnd.GetValue() > k32BitSize) { + return; + } + auto &extDefOpnd = static_cast(insn.GetOperand(kInsnFirstOpnd)); + auto &lslUseOpnd = static_cast(useInsn->GetOperand(kInsnSecondOpnd)); + regno_t extDefRegNO = extDefOpnd.GetRegisterNumber(); + regno_t lslUseRegNO = lslUseOpnd.GetRegisterNumber(); + if (extDefRegNO != lslUseRegNO || IfOperandIsLiveAfterInsn(extDefOpnd, *useInsn)) { + return; + } + + MOperator mopNew = (curMop == MOP_xsxtw64 ? MOP_xsbfizrri6i6 : MOP_xubfizrri6i6); + auto &extUseOpnd = static_cast(insn.GetOperand(kInsnSecondOpnd)); + auto &lslDefOpnd = static_cast(useInsn->GetOperand(kInsnFirstOpnd)); + ImmOperand &newImmOpnd = static_cast(cgFunc)->CreateImmOperand(k32BitSize, k6BitSize, false); + Insn &newInsn = cgFunc->GetInsnBuilder()->BuildInsn(mopNew, lslDefOpnd, extUseOpnd, lslImmOpnd, newImmOpnd); + bb.RemoveInsn(*useInsn); + bb.ReplaceInsn(insn, newInsn); + optSuccess = true; +} + bool ComplexExtendWordLslAArch64::IsExtendWordLslPattern(const Insn &insn) const { Insn *nextInsn = insn.GetNext(); @@ -6020,4 +8424,133 @@ void ComplexExtendWordLslAArch64::Run(BB &bb, Insn &insn) bb.RemoveInsn(*nextInsn); bb.ReplaceInsn(insn, newInsnSbfiz); } + +bool AddCmpZeroPatternSSA::CheckCondition(Insn &insn) +{ + MOperator curMop = insn.GetMachineOpcode(); + if (curMop != MOP_wcmpri && curMop != MOP_xcmpri) { + return false; + } + auto &immOpnd = static_cast(insn.GetOperand(kInsnThirdOpnd)); + if (!immOpnd.IsZero()) { + return false; + } + + auto &ccReg = static_cast(insn.GetOperand(kInsnSecondOpnd)); + prevAddInsn = ssaInfo->GetDefInsn(ccReg); + if (prevAddInsn == nullptr) { + return false; + } + MOperator prevAddMop = prevAddInsn->GetMachineOpcode(); + if (prevAddMop != MOP_xaddrrr && prevAddMop != MOP_xaddrri12 && prevAddMop != MOP_waddrrr && + prevAddMop != MOP_waddrri12 && prevAddMop != MOP_xaddrrrs && prevAddMop != MOP_waddrrrs) { + return false; + } + Insn *nextInsn = insn.GetNext(); + while (nextInsn != nullptr) { + if (!nextInsn->IsMachineInstruction()) { + return false; + } + for (uint32 i = 0; i < nextInsn->GetOperandSize(); ++i) { + if (nextInsn->GetDesc()->GetOpndDes(i) == &OpndDesc::Cond) { + CondOperand &cond = static_cast(nextInsn->GetOperand(i)); + if (cond.GetCode() == CC_EQ) { + return true; + } else { + return false; + } + } + } + nextInsn = nextInsn->GetNext(); + } + return false; +} + +void AddCmpZeroPatternSSA::Run(BB &bb, Insn &insn) +{ + if (!CheckCondition(insn)) { + return; + } + bool isShiftAdd = false; + MOperator prevAddMop = prevAddInsn->GetMachineOpcode(); + MOperator newAddMop = GetMopUpdateAPSR(prevAddMop, isShiftAdd); + DEBUG_ASSERT(newAddMop != MOP_undef, "unknown Add code"); + /* + * Since new opnd can not be defined in SSA ReplaceInsn, we should avoid pattern matching again. + * For "adds" can only be inserted in this phase, so we could do a simple check. + */ + Insn *nextInsn = insn.GetNext(); + while (nextInsn != nullptr) { + if (!nextInsn->IsMachineInstruction()) { + nextInsn = nextInsn->GetNext(); + continue; + } + MOperator nextMop = nextInsn->GetMachineOpcode(); + if (nextMop == newAddMop) { + return; + } + nextInsn = nextInsn->GetNext(); + } + + Insn *newInsn = nullptr; + Operand &rflag = insn.GetOperand(kInsnFirstOpnd); + if (isShiftAdd) { + newInsn = &cgFunc->GetInsnBuilder()->BuildInsn( + newAddMop, rflag, prevAddInsn->GetOperand(kInsnFirstOpnd), prevAddInsn->GetOperand(kInsnSecondOpnd), + prevAddInsn->GetOperand(kInsnThirdOpnd), prevAddInsn->GetOperand(kInsnFourthOpnd)); + } else { + newInsn = &cgFunc->GetInsnBuilder()->BuildInsn(newAddMop, rflag, prevAddInsn->GetOperand(kInsnFirstOpnd), + prevAddInsn->GetOperand(kInsnSecondOpnd), + prevAddInsn->GetOperand(kInsnThirdOpnd)); + } + bb.InsertInsnAfter(insn, *newInsn); + /* update ssa info */ + auto *a64SSAInfo = static_cast(ssaInfo); + a64SSAInfo->CreateNewInsnSSAInfo(*newInsn); + SetCurrInsn(newInsn); + + /* dump pattern info */ + if (CG_PEEP_DUMP) { + std::vector prevs; + prevs.emplace_back(prevAddInsn); + prevs.emplace_back(&insn); + DumpAfterPattern(prevs, newInsn, nullptr); + } +} + +bool DeleteAndBeforeRevStrPattern::CheckCondition(Insn &insn) +{ + auto &andImmOpnd = static_cast(insn.GetOperand(kInsnThirdOpnd)); + uint64 andImmValue = static_cast(andImmOpnd.GetValue()); + if (andImmValue != 0xFFFFULL) { // 0xFFFF : 16bits mask + return false; + } + auto &insnDefOpnd = insn.GetOperand(kInsnFirstOpnd); + Insn *revInsn = insn.GetNextMachineInsn(); + if ((revInsn == nullptr) || (revInsn->GetMachineOpcode() != MOP_wrevrr16) || + (!RegOperand::IsSameRegNO(insnDefOpnd, revInsn->GetOperand(kInsnSecondOpnd)))) { + return false; + } + Insn *strInsn = revInsn->GetNextMachineInsn(); + if ((strInsn == nullptr) || (strInsn->GetMachineOpcode() != MOP_wstrh) || + (!RegOperand::IsSameRegNO(revInsn->GetOperand(kInsnFirstOpnd), strInsn->GetOperand(kInsnFirstOpnd)))) { + return false; + } + if ((!RegOperand::IsSameRegNO(insnDefOpnd, strInsn->GetOperand(kInsnFirstOpnd))) && + IfOperandIsLiveAfterInsn(static_cast(insnDefOpnd), *strInsn)) { + return false; + } + return true; +} + +void DeleteAndBeforeRevStrPattern::Run(BB &bb, Insn &insn) +{ + if (!CheckCondition(insn)) { + return; + } + auto &insnUseOpnd = insn.GetOperand(kInsnSecondOpnd); + Insn *nextInsn = insn.GetNextMachineInsn(); + nextInsn->SetOperand(kInsnSecondOpnd, insnUseOpnd); + bb.RemoveInsn(insn); +} } /* namespace maplebe */ diff --git a/ecmascript/compiler/codegen/maple/maple_be/src/cg/cfgo.cpp b/ecmascript/compiler/codegen/maple/maple_be/src/cg/cfgo.cpp index 1267a4f40327ee3fbe0bc491cbf7681ca77fcf2d..5f647ba41d427bd5178beeaafdb725623a904a75 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/src/cg/cfgo.cpp +++ b/ecmascript/compiler/codegen/maple/maple_be/src/cg/cfgo.cpp @@ -16,6 +16,7 @@ #include "cfgo.h" #include "cgbb.h" #include "cg.h" +#include "loop.h" #include "mpl_logging.h" /* @@ -31,6 +32,17 @@ using namespace maple; #define CFGO_DUMP_NEWPM CG_DEBUG_FUNC(f) +// find for a BB that can fallthru to curBB, which has only one BB at most. +static BB *FindPredFallthruBB(const BB &curBB) +{ + for (auto *pred : curBB.GetPreds()) { + if (pred->GetKind() == BB::kBBFallthru) { + return pred; + } + } + return nullptr; +} + /* return true if to is put after from and there is no real insns between from and to, */ bool ChainingPattern::NoInsnBetween(const BB &from, const BB &to) const { @@ -46,9 +58,10 @@ bool ChainingPattern::NoInsnBetween(const BB &from, const BB &to) const /* return true if insns in bb1 and bb2 are the same except the last goto insn. */ bool ChainingPattern::DoSameThing(const BB &bb1, const Insn &last1, const BB &bb2, const Insn &last2) const { - const Insn *insn1 = bb1.GetFirstInsn(); - const Insn *insn2 = bb2.GetFirstInsn(); - while (insn1 != nullptr && insn1 != last1.GetNext() && insn2 != nullptr && insn2 != last2.GetNext()) { + const Insn *insn1 = bb1.GetFirstMachineInsn(); + const Insn *insn2 = bb2.GetFirstMachineInsn(); + while (insn1 != nullptr && insn1 != last1.GetNextMachineInsn() && insn2 != nullptr && + insn2 != last2.GetNextMachineInsn()) { if (!insn1->IsMachineInstruction()) { insn1 = insn1->GetNext(); continue; @@ -71,10 +84,10 @@ bool ChainingPattern::DoSameThing(const BB &bb1, const Insn &last1, const BB &bb return false; } } - insn1 = insn1->GetNext(); - insn2 = insn2->GetNext(); + insn1 = insn1->GetNextMachineInsn(); + insn2 = insn2->GetNextMachineInsn(); } - return (insn1 == last1.GetNext() && insn2 == last2.GetNext()); + return (insn1 == last1.GetNextMachineInsn() && insn2 == last2.GetNextMachineInsn()); } /* @@ -90,6 +103,9 @@ bool ChainingPattern::MergeFallthuBB(BB &curBB) !cgFunc->GetTheCFG()->CanMerge(curBB, *sucBB)) { return false; } + if (curBB.IsAtomicBuiltInBB() || sucBB->IsAtomicBuiltInBB()) { + return false; + } Log(curBB.GetId()); if (checkOnly) { return false; @@ -135,9 +151,16 @@ bool ChainingPattern::MoveSuccBBAsCurBBNext(BB &curBB, BB &sucBB) sucBB.SetNext(curBB.GetNext()); DEBUG_ASSERT(curBB.GetNext() != nullptr, "current goto BB will not be the last bb"); curBB.GetNext()->SetPrev(&sucBB); + if (sucBB.GetId() == cgFunc->GetLastBB()->GetId()) { + cgFunc->SetLastBB(*(sucBB.GetPrev())); + } else if (curBB.GetId() == cgFunc->GetLastBB()->GetId()) { + cgFunc->SetLastBB(sucBB); + } sucBB.SetPrev(&curBB); curBB.SetNext(&sucBB); - curBB.RemoveInsn(*curBB.GetLastInsn()); + if (curBB.GetLastMachineInsn() != nullptr) { + curBB.RemoveInsn(*curBB.GetLastMachineInsn()); + } curBB.SetKind(BB::kBBFallthru); return true; } @@ -150,12 +173,13 @@ bool ChainingPattern::RemoveGotoInsn(BB &curBB, BB &sucBB) } if (&sucBB != curBB.GetNext()) { DEBUG_ASSERT(curBB.GetNext() != nullptr, "nullptr check"); - curBB.RemoveSuccs(sucBB); - curBB.PushBackSuccs(*curBB.GetNext()); + curBB.ReplaceSucc(sucBB, *curBB.GetNext()); curBB.GetNext()->PushBackPreds(curBB); sucBB.RemovePreds(curBB); } - curBB.RemoveInsn(*curBB.GetLastInsn()); + if (curBB.GetLastMachineInsn() != nullptr) { + curBB.RemoveInsn(*curBB.GetLastMachineInsn()); + } curBB.SetKind(BB::kBBFallthru); return true; } @@ -166,7 +190,7 @@ bool ChainingPattern::ClearCurBBAndResetTargetBB(BB &curBB, BB &sucBB) return false; } Insn *brInsn = nullptr; - for (brInsn = curBB.GetLastInsn(); brInsn != nullptr; brInsn = brInsn->GetPrev()) { + for (brInsn = curBB.GetLastMachineInsn(); brInsn != nullptr; brInsn = brInsn->GetPrev()) { if (brInsn->IsUnCondBranch()) { break; } @@ -174,18 +198,23 @@ bool ChainingPattern::ClearCurBBAndResetTargetBB(BB &curBB, BB &sucBB) DEBUG_ASSERT(brInsn != nullptr, "goto BB has no branch"); BB *newTarget = sucBB.GetPrev(); DEBUG_ASSERT(newTarget != nullptr, "get prev bb failed in ChainingPattern::ClearCurBBAndResetTargetBB"); - Insn *last1 = newTarget->GetLastInsn(); + Insn *last1 = newTarget->GetLastMachineInsn(); if (newTarget->GetKind() == BB::kBBGoto) { Insn *br = nullptr; - for (br = newTarget->GetLastInsn(); br != newTarget->GetFirstInsn()->GetPrev(); br = br->GetPrev()) { + for (br = newTarget->GetLastMachineInsn(); + newTarget->GetFirstInsn() != nullptr && br != newTarget->GetFirstInsn()->GetPrev(); br = br->GetPrev()) { if (br->IsUnCondBranch()) { break; } } DEBUG_ASSERT(br != nullptr, "goto BB has no branch"); - last1 = br->GetPrev(); + last1 = br->GetPreviousMachineInsn(); } - if (last1 == nullptr || !DoSameThing(*newTarget, *last1, curBB, *brInsn->GetPrev())) { + if (last1 == nullptr) { + return false; + } + if (brInsn->GetPreviousMachineInsn() && + !DoSameThing(*newTarget, *last1, curBB, *brInsn->GetPreviousMachineInsn())) { return false; } @@ -222,17 +251,21 @@ bool ChainingPattern::ClearCurBBAndResetTargetBB(BB &curBB, BB &sucBB) */ bool ChainingPattern::Optimize(BB &curBB) { + if (curBB.IsUnreachable()) { + return false; + } + if (curBB.GetKind() == BB::kBBFallthru) { return MergeFallthuBB(curBB); } if (curBB.GetKind() == BB::kBBGoto && !curBB.IsEmpty()) { - Insn *last = curBB.GetLastInsn(); + Insn *last = curBB.GetLastMachineInsn(); if (last->IsTailCall()) { return false; } - BB *sucBB = cgFunc->GetTheCFG()->GetTargetSuc(curBB); + BB *sucBB = CGCFG::GetTargetSuc(curBB); /* * BB2 can be merged into BB1, if * 1. BB1 ends with a goto; @@ -248,12 +281,19 @@ bool ChainingPattern::Optimize(BB &curBB) } if (sucBB->GetKind() == BB::kBBGoto && !IsLabelInLSDAOrSwitchTable(sucBB->GetLabIdx()) && cgFunc->GetTheCFG()->CanMerge(curBB, *sucBB)) { + // BB9(curBB) BB1 + // \ / + // BB5(sucBB, gotoBB) + // for this case, should not merge BB5, BB9 + if (sucBB->GetPreds().size() > 1) { + return false; + } return MergeGotoBB(curBB, *sucBB); } else if (sucBB != &curBB && curBB.GetNext() != sucBB && sucBB != cgFunc->GetLastBB() && !sucBB->IsPredecessor(*sucBB->GetPrev()) && !(sucBB->GetNext() != nullptr && sucBB->GetNext()->IsPredecessor(*sucBB)) && !IsLabelInLSDAOrSwitchTable(sucBB->GetLabIdx()) && sucBB->GetEhSuccs().empty() && - sucBB->GetKind() != BB::kBBThrow) { + sucBB->GetKind() != BB::kBBThrow && curBB.GetNext() != nullptr) { return MoveSuccBBAsCurBBNext(curBB, *sucBB); } /* @@ -314,10 +354,11 @@ bool SequentialJumpPattern::Optimize(BB &curBB) return false; } if (curBB.GetKind() == BB::kBBGoto && !curBB.IsEmpty()) { - BB *sucBB = cgFunc->GetTheCFG()->GetTargetSuc(curBB); + BB *sucBB = CGCFG::GetTargetSuc(curBB); CHECK_FATAL(sucBB != nullptr, "sucBB is null in SequentialJumpPattern::Optimize"); - BB *tragetBB = CGCFG::GetTargetSuc(*sucBB); - if ((sucBB != &curBB) && sucBB->IsSoloGoto() && tragetBB != nullptr && tragetBB != sucBB) { + BB *targetBB = CGCFG::GetTargetSuc(*sucBB); + if ((sucBB != &curBB) && sucBB->IsSoloGoto() && targetBB != nullptr && targetBB != sucBB && + !HasInvalidPred(*sucBB)) { Log(curBB.GetId()); if (checkOnly) { return false; @@ -328,13 +369,35 @@ bool SequentialJumpPattern::Optimize(BB &curBB) } } else if (curBB.GetKind() == BB::kBBIf) { for (BB *sucBB : curBB.GetSuccs()) { - BB *tragetBB = CGCFG::GetTargetSuc(*sucBB); - if (sucBB != curBB.GetNext() && sucBB->IsSoloGoto() && tragetBB != nullptr && tragetBB != sucBB) { + BB *sucTargetBB = CGCFG::GetTargetSuc(*sucBB); + if (sucBB != curBB.GetNext() && sucBB->IsSoloGoto() && sucTargetBB != nullptr && sucTargetBB != sucBB && + !HasInvalidPred(*sucBB)) { Log(curBB.GetId()); if (checkOnly) { return false; } - cgFunc->GetTheCFG()->RetargetJump(*sucBB, curBB); + // e.g. + // BB12[if] (curBB) + // beq label:11 + // / \ + // / BB25[if] (label:11) + // \ / + // BB13[goto] (sucBB) + // | + // BB6[ft] (targetBB) (label: 6) + // For the above case, the ifBB can not modify the target label of the conditional jump insn, + // because the target of the conditional jump insn is the other succBB(BB25). + BB *ifTargetBB = CGCFG::GetTargetSuc(curBB); + CHECK_NULL_FATAL(ifTargetBB); + // In addition, if the targetBB(ifTargetBB) of the ifBB is not the gotoBB(sucBB), and the + // targetBB(sucTargetBB) of the sucBB is not its next, we can not do the optimization, because it will + // change the layout of the sucTargetBB, and it does not necessarily improve performance. + if (ifTargetBB != sucBB && sucBB->GetNext() != sucTargetBB) { + return false; + } + if (ifTargetBB == sucBB) { + cgFunc->GetTheCFG()->RetargetJump(*sucBB, curBB); + } SkipSucBB(curBB, *sucBB); return true; } @@ -342,8 +405,8 @@ bool SequentialJumpPattern::Optimize(BB &curBB) } else if (curBB.GetKind() == BB::kBBRangeGoto) { bool changed = false; for (BB *sucBB : curBB.GetSuccs()) { - if (sucBB != curBB.GetNext() && sucBB->IsSoloGoto() && - cgFunc->GetTheCFG()->GetTargetSuc(*sucBB) != nullptr) { + if (sucBB != curBB.GetNext() && sucBB->IsSoloGoto() && !HasInvalidPred(*sucBB) && + CGCFG::GetTargetSuc(*sucBB) != nullptr) { Log(curBB.GetId()); if (checkOnly) { return false; @@ -358,9 +421,9 @@ bool SequentialJumpPattern::Optimize(BB &curBB) return false; } -void SequentialJumpPattern::UpdateSwitchSucc(BB &curBB, BB &sucBB) +void SequentialJumpPattern::UpdateSwitchSucc(BB &curBB, BB &sucBB) const { - BB *gotoTarget = cgFunc->GetTheCFG()->GetTargetSuc(sucBB); + BB *gotoTarget = CGCFG::GetTargetSuc(sucBB); CHECK_FATAL(gotoTarget != nullptr, "gotoTarget is null in SequentialJumpPattern::UpdateSwitchSucc"); const MapleVector &labelVec = curBB.GetRangeGotoLabelVec(); bool isPred = false; @@ -372,7 +435,7 @@ void SequentialJumpPattern::UpdateSwitchSucc(BB &curBB, BB &sucBB) } for (size_t i = 0; i < labelVec.size(); ++i) { if (labelVec[i] == sucBB.GetLabIdx()) { - curBB.SetRangeGotoLabel(i, gotoTarget->GetLabIdx()); + curBB.SetRangeGotoLabel(static_cast(i), gotoTarget->GetLabIdx()); } } cgFunc->UpdateEmitSt(curBB, sucBB.GetLabIdx(), gotoTarget->GetLabIdx()); @@ -422,21 +485,101 @@ void SequentialJumpPattern::UpdateSwitchSucc(BB &curBB, BB &sucBB) } } +bool SequentialJumpPattern::HasInvalidPred(BB &sucBB) const +{ + for (auto predIt = sucBB.GetPredsBegin(); predIt != sucBB.GetPredsEnd(); ++predIt) { + if ((*predIt)->GetKind() != BB::kBBGoto && (*predIt)->GetKind() != BB::kBBIf && + (*predIt)->GetKind() != BB::kBBRangeGoto && (*predIt)->GetKind() != BB::kBBFallthru) { + return true; + } + if ((*predIt)->GetKind() == BB::kBBIf) { + BB *ifTargetBB = CGCFG::GetTargetSuc(**predIt); + CHECK_NULL_FATAL(ifTargetBB); + BB *sucTargetBB = CGCFG::GetTargetSuc(sucBB); + CHECK_NULL_FATAL(sucTargetBB); + if (ifTargetBB != &sucBB && sucBB.GetNext() != sucTargetBB) { + return true; + } + } + if ((*predIt)->GetKind() == BB::kBBIf || (*predIt)->GetKind() == BB::kBBRangeGoto) { + if ((*predIt)->GetNext() == &sucBB) { + return true; + } + } else if ((*predIt)->GetKind() == BB::kBBGoto) { + if (*predIt == &sucBB || (*predIt)->IsEmpty()) { + return true; + } + } + } + return false; +} + /* * preCond: * sucBB is one of curBB's successor. * * Change curBB's successor to sucBB's successor */ -void SequentialJumpPattern::SkipSucBB(BB &curBB, BB &sucBB) +void SequentialJumpPattern::SkipSucBB(BB &curBB, BB &sucBB) const { - BB *gotoTarget = cgFunc->GetTheCFG()->GetTargetSuc(sucBB); + BB *gotoTarget = CGCFG::GetTargetSuc(sucBB); CHECK_FATAL(gotoTarget != nullptr, "gotoTarget is null in SequentialJumpPattern::SkipSucBB"); - curBB.RemoveSuccs(sucBB); - curBB.PushBackSuccs(*gotoTarget); + curBB.ReplaceSucc(sucBB, *gotoTarget); sucBB.RemovePreds(curBB); gotoTarget->PushBackPreds(curBB); + // If the sucBB needs to be skipped, all preds of the sucBB must skip it and update cfg info. + // e.g. + // BB3[if] (curBB) + // / \ + // / BB6[if] (not care) + // \ / + // BB10[goto] (sucBB) + // | + // BB8 (gotoTarget) + for (auto *predBB : sucBB.GetPreds()) { + if (predBB->GetKind() == BB::kBBGoto) { + cgFunc->GetTheCFG()->RetargetJump(sucBB, *predBB); + } else if (predBB->GetKind() == BB::kBBIf) { + BB *ifTargetBB = CGCFG::GetTargetSuc(*predBB); + if (ifTargetBB == &sucBB) { + cgFunc->GetTheCFG()->RetargetJump(sucBB, *predBB); + } + } else if (predBB->GetKind() == BB::kBBFallthru) { + // e.g. + // (curBB) BB70[goto] BB27[if] + // \ / \ + // \ / \ + // \ BB71[ft] (iterPredBB) \ + // \ / \ + // BB48[goto] (sucBB) BB28[ft] + // | / + // | / + // BB29[if] (gotoTarget) + ASSERT_NOT_NULL(cgFunc->GetTheCFG()->GetInsnModifier()); + cgFunc->GetTheCFG()->GetInsnModifier()->ModifyFathruBBToGotoBB(*predBB, gotoTarget->GetLabIdx()); + } else if (predBB->GetKind() == BB::kBBRangeGoto) { + UpdateSwitchSucc(*predBB, sucBB); + } + predBB->ReplaceSucc(sucBB, *gotoTarget); + sucBB.RemovePreds(*predBB); + gotoTarget->PushBackPreds(*predBB); + } cgFunc->GetTheCFG()->FlushUnReachableStatusAndRemoveRelations(sucBB, *cgFunc); + // LastBB cannot be removed from the preds of succBB by FlushUnReachableStatusAndRemoveRelations, Why? + // We'll do a separate process below for the case that sucBB is LastBB. + if (sucBB.GetKind() == BB::kBBGoto && &sucBB == cgFunc->GetLastBB()) { + // gotoBB has only one succ. + DEBUG_ASSERT(sucBB.GetSuccsSize() == 1, "invalid gotoBB"); + sucBB.SetUnreachable(true); + sucBB.SetFirstInsn(nullptr); + sucBB.SetLastInsn(nullptr); + gotoTarget->RemovePreds(sucBB); + sucBB.RemoveSuccs(*gotoTarget); + } + // Remove the unreachableBB which has been skipped + if (sucBB.IsUnreachable()) { + cgFunc->GetTheCFG()->RemoveBB(sucBB); + } } /* @@ -458,16 +601,17 @@ void FlipBRPattern::RelocateThrowBB(BB &curBB) CGCFG *theCFG = cgFunc->GetTheCFG(); CHECK_FATAL(theCFG != nullptr, "nullptr check"); BB *retBB = theCFG->FindLastRetBB(); + retBB = (retBB == nullptr ? cgFunc->GetLastBB() : retBB); CHECK_FATAL(retBB != nullptr, "must have a return BB"); if (ftBB->GetKind() != BB::kBBThrow || !ftBB->GetEhSuccs().empty() || IsLabelInLSDAOrSwitchTable(ftBB->GetLabIdx()) || !retBB->GetEhSuccs().empty()) { return; } - BB *brBB = theCFG->GetTargetSuc(curBB); + BB *brBB = CGCFG::GetTargetSuc(curBB); if (brBB != ftBB->GetNext()) { return; } - +#ifndef ONLY_C EHFunc *ehFunc = cgFunc->GetEHFunc(); if (ehFunc != nullptr && ehFunc->GetLSDACallSiteTable() != nullptr) { const MapleVector &callsiteTable = ehFunc->GetLSDACallSiteTable()->GetCallSiteTable(); @@ -488,6 +632,7 @@ void FlipBRPattern::RelocateThrowBB(BB &curBB) } } } +#endif /* get branch insn of curBB */ Insn *curBBBranchInsn = theCFG->FindLastCondBrInsn(curBB); CHECK_FATAL(curBBBranchInsn != nullptr, "curBB(it is a kBBif) has no branch"); @@ -501,6 +646,7 @@ void FlipBRPattern::RelocateThrowBB(BB &curBB) /* move ftBB after retBB */ curBB.SetNext(brBB); + CHECK_NULL_FATAL(brBB); brBB->SetPrev(&curBB); retBB->GetNext()->SetPrev(ftBB); @@ -523,14 +669,23 @@ void FlipBRPattern::RelocateThrowBB(BB &curBB) * ftBB * targetBB * - * 2. relocate throw BB in RelocateThrowBB() + * 2. loopHeaderBB: loopHeaderBB: + * ... ... + * cond_br loopExit: cond_br loopHeaderBB + * ftBB: ftBB: + * goto loopHeaderBB: goto loopExit + * + * 3. relocate throw BB in RelocateThrowBB() */ bool FlipBRPattern::Optimize(BB &curBB) { + if (curBB.IsUnreachable()) { + return false; + } if (curBB.GetKind() == BB::kBBIf && !curBB.IsEmpty()) { BB *ftBB = curBB.GetNext(); DEBUG_ASSERT(ftBB != nullptr, "ftBB is null in FlipBRPattern::Optimize"); - BB *brBB = cgFunc->GetTheCFG()->GetTargetSuc(curBB); + BB *brBB = CGCFG::GetTargetSuc(curBB); DEBUG_ASSERT(brBB != nullptr, "brBB is null in FlipBRPattern::Optimize"); /* Check if it can be optimized */ if (ftBB->GetKind() == BB::kBBGoto && ftBB->GetNext() == brBB) { @@ -538,7 +693,7 @@ bool FlipBRPattern::Optimize(BB &curBB) return false; } Insn *curBBBranchInsn = nullptr; - for (curBBBranchInsn = curBB.GetLastInsn(); curBBBranchInsn != nullptr; + for (curBBBranchInsn = curBB.GetLastMachineInsn(); curBBBranchInsn != nullptr; curBBBranchInsn = curBBBranchInsn->GetPrev()) { if (curBBBranchInsn->IsBranch()) { break; @@ -546,7 +701,7 @@ bool FlipBRPattern::Optimize(BB &curBB) } DEBUG_ASSERT(curBBBranchInsn != nullptr, "FlipBRPattern: curBB has no branch"); Insn *brInsn = nullptr; - for (brInsn = ftBB->GetLastInsn(); brInsn != nullptr; brInsn = brInsn->GetPrev()) { + for (brInsn = ftBB->GetLastMachineInsn(); brInsn != nullptr; brInsn = brInsn->GetPrev()) { if (brInsn->IsUnCondBranch()) { break; } @@ -578,12 +733,10 @@ bool FlipBRPattern::Optimize(BB &curBB) it = curBB.GetSuccsBegin(); CHECK_FATAL(*it != nullptr, "nullptr check"); if (*it == brBB) { - curBB.EraseSuccs(it); - curBB.PushBackSuccs(*tgtBB); + curBB.ReplaceSucc(it, *tgtBB); } else { ++it; - curBB.EraseSuccs(it); - curBB.PushFrontSuccs(*tgtBB); + curBB.ReplaceSucc(it, *tgtBB); } for (it = tgtBB->GetPredsBegin(); it != tgtBB->GetPredsEnd(); ++it) { if (*it == ftBB) { @@ -623,6 +776,46 @@ bool FlipBRPattern::Optimize(BB &curBB) ftBB->RemoveInsn(*brInsn); ftBB->SetKind(BB::kBBFallthru); } + } else if (GetPhase() == kCfgoPostRegAlloc && ftBB->GetKind() == BB::kBBGoto && curBB.GetLoop() != nullptr && + curBB.GetLoop() == ftBB->GetLoop() && ftBB->IsSoloGoto() && + ftBB->GetLoop()->GetHeader() == *(ftBB->GetSuccsBegin()) && + !curBB.GetLoop()->IsBBLoopMember((curBB.GetSuccs().front() == ftBB) ? curBB.GetSuccs().back() + : curBB.GetSuccs().front())) { + Insn *curBBBranchInsn = nullptr; + for (curBBBranchInsn = curBB.GetLastMachineInsn(); curBBBranchInsn != nullptr; + curBBBranchInsn = curBBBranchInsn->GetPrev()) { + if (curBBBranchInsn->IsBranch()) { + break; + } + } + DEBUG_ASSERT(curBBBranchInsn != nullptr, "FlipBRPattern: curBB has no branch"); + Insn *brInsn = nullptr; + for (brInsn = ftBB->GetLastMachineInsn(); brInsn != nullptr; brInsn = brInsn->GetPrev()) { + if (brInsn->IsUnCondBranch()) { + break; + } + } + DEBUG_ASSERT(brInsn != nullptr, "FlipBRPattern: ftBB has no branch"); + uint32 condTargetIdx = GetJumpTargetIdx(*curBBBranchInsn); + LabelOperand &condTarget = static_cast(curBBBranchInsn->GetOperand(condTargetIdx)); + MOperator mOp = FlipConditionOp(curBBBranchInsn->GetMachineOpcode()); + if (mOp == 0) { + return false; + } + uint32 gotoTargetIdx = GetJumpTargetIdx(*brInsn); + LabelOperand &gotoTarget = static_cast(brInsn->GetOperand(gotoTargetIdx)); + curBBBranchInsn->SetMOP(cgFunc->GetCG()->GetTargetMd(mOp)); + curBBBranchInsn->SetOperand(condTargetIdx, gotoTarget); + brInsn->SetOperand(gotoTargetIdx, condTarget); + auto it = ftBB->GetSuccsBegin(); + BB *loopHeadBB = *it; + curBB.ReplaceSucc(*brBB, *loopHeadBB); + brBB->RemovePreds(curBB); + ftBB->ReplaceSucc(*loopHeadBB, *brBB); + loopHeadBB->RemovePreds(*ftBB); + + loopHeadBB->PushBackPreds(curBB); + brBB->PushBackPreds(*ftBB); } else { RelocateThrowBB(curBB); } @@ -633,25 +826,47 @@ bool FlipBRPattern::Optimize(BB &curBB) /* remove a basic block that contains nothing */ bool EmptyBBPattern::Optimize(BB &curBB) { - if (curBB.IsUnreachable()) { + // Can not remove the BB whose address is referenced by adrp_label insn + if (curBB.IsUnreachable() || curBB.IsAdrpLabel()) { return false; } - /* Empty bb but do not have cleanup label. */ - if (curBB.GetPrev() != nullptr && curBB.GetFirstStmt() != cgFunc->GetCleanupLabel() && - curBB.GetFirstInsn() == nullptr && curBB.GetLastInsn() == nullptr && &curBB != cgFunc->GetLastBB() && - curBB.GetKind() != BB::kBBReturn && !IsLabelInLSDAOrSwitchTable(curBB.GetLabIdx())) { + /* Empty bb and it's not a cleanupBB/returnBB/lastBB/catchBB. */ + if (curBB.GetPrev() == nullptr || curBB.IsCleanup() || &curBB == cgFunc->GetLastBB() || + curBB.GetKind() == BB::kBBReturn || IsLabelInLSDAOrSwitchTable(curBB.GetLabIdx())) { + return false; + } + if (curBB.GetFirstInsn() == nullptr && curBB.GetLastInsn() == nullptr) { + // empty BB Log(curBB.GetId()); if (checkOnly) { return false; } - BB *sucBB = cgFunc->GetTheCFG()->GetTargetSuc(curBB); - if (sucBB == nullptr || sucBB->GetFirstStmt() == cgFunc->GetCleanupLabel()) { + BB *sucBB = CGCFG::GetTargetSuc(curBB); + if (sucBB == nullptr || sucBB->IsCleanup()) { return false; } cgFunc->GetTheCFG()->RemoveBB(curBB); /* removeBB may do nothing. since no need to repeat, always ret false here. */ return false; + } else if (!curBB.HasMachineInsn()) { + // BB only has dbg insn + Log(curBB.GetId()); + if (checkOnly) { + return false; + } + BB *sucBB = CGCFG::GetTargetSuc(curBB); + if (sucBB == nullptr || sucBB->IsCleanup()) { + return false; + } + // For Now We try to sink first conservatively. + // All dbg insns should not be dropped. Later hoist or copy case should be considered. + if (curBB.NumSuccs() == 1) { + BB *succBB = curBB.GetSuccs().front(); + succBB->InsertAtBeginning(curBB); + cgFunc->GetTheCFG()->RemoveBB(curBB); + } + return false; } return false; } @@ -670,7 +885,8 @@ bool UnreachBBPattern::Optimize(BB &curBB) } /* if curBB in exitbbsvec,return false. */ if (cgFunc->IsExitBB(curBB)) { - curBB.SetUnreachable(false); + // In C some bb follow noreturn calls should remain unreachable + curBB.SetUnreachable(cgFunc->GetMirModule().GetSrcLang() == kSrcLangC); return false; } @@ -678,9 +894,10 @@ bool UnreachBBPattern::Optimize(BB &curBB) return false; } +#ifndef ONLY_C EHFunc *ehFunc = cgFunc->GetEHFunc(); /* if curBB InLSDA ,replace curBB's label with nextReachableBB before remove it. */ - if (ehFunc != nullptr && ehFunc->NeedFullLSDA() && cgFunc->GetTheCFG()->InLSDA(curBB.GetLabIdx(), *ehFunc)) { + if (ehFunc != nullptr && ehFunc->NeedFullLSDA() && CGCFG::InLSDA(curBB.GetLabIdx(), ehFunc)) { /* find nextReachableBB */ BB *nextReachableBB = nullptr; for (BB *bb = &curBB; bb != nullptr; bb = bb->GetNext()) { @@ -698,16 +915,20 @@ bool UnreachBBPattern::Optimize(BB &curBB) ehFunc->GetLSDACallSiteTable()->UpdateCallSite(curBB, *nextReachableBB); } +#endif - if (curBB.GetSuccs().empty() && curBB.GetEhSuccs().empty()) { - return false; - } - + // Indicates whether the curBB is removed + bool isRemoved = true; if (curBB.GetPrev() != nullptr) { curBB.GetPrev()->SetNext(curBB.GetNext()); } if (curBB.GetNext() != nullptr) { curBB.GetNext()->SetPrev(curBB.GetPrev()); + } else { + cgFunc->SetLastBB(*(curBB.GetPrev())); + } + if (cgFunc->GetFirstBB() == cgFunc->GetLastBB() && cgFunc->GetFirstBB() != nullptr) { + isRemoved = false; } /* flush after remove; */ @@ -721,7 +942,10 @@ bool UnreachBBPattern::Optimize(BB &curBB) } curBB.ClearSuccs(); curBB.ClearEhSuccs(); - return true; + // return always be false + if (!isRemoved) { + return false; + } } return false; } @@ -744,6 +968,9 @@ bool UnreachBBPattern::Optimize(BB &curBB) */ bool DuplicateBBPattern::Optimize(BB &curBB) { + if (!cgFunc->IsAfterRegAlloc()) { + return false; + } if (curBB.IsUnreachable()) { return false; } @@ -757,23 +984,23 @@ bool DuplicateBBPattern::Optimize(BB &curBB) return false; } -#if TARGARM32 - FOR_BB_INSNS(insn, (&curBB)) { +#if defined(TARGARM32) && TARGARM32 + FOR_BB_INSNS(insn, (&curBB)) + { if (insn->IsPCLoad() || insn->IsClinit()) { return false; } } #endif - /* It is possible curBB jump to itself */ - uint32 numPreds = curBB.NumPreds(); + /* It is possible curBB jump to itself, no benefit from duplicate */ for (BB *bb : curBB.GetPreds()) { if (bb == &curBB) { - numPreds--; + return false; } } - if (numPreds > 1 && cgFunc->GetTheCFG()->GetTargetSuc(curBB) != nullptr && - cgFunc->GetTheCFG()->GetTargetSuc(curBB)->NumPreds() > 1) { + uint32 numPreds = curBB.NumPreds(); + if (numPreds > 1 && CGCFG::GetTargetSuc(curBB) != nullptr && CGCFG::GetTargetSuc(curBB)->NumPreds() > 1) { std::vector candidates; for (BB *bb : curBB.GetPreds()) { if (bb->GetKind() == BB::kBBGoto && bb->GetNext() != &curBB && bb != &curBB && !bb->IsEmpty()) { @@ -800,7 +1027,11 @@ bool DuplicateBBPattern::Optimize(BB &curBB) continue; } bb->RemoveInsn(*bb->GetLastInsn()); - FOR_BB_INSNS(insn, (&curBB)) { + FOR_BB_INSNS(insn, (&curBB)) + { + if (!insn->IsMachineInstruction() && !(bb->IsCloseTo(curBB) && insn->IsDbgLine())) { + continue; + } Insn *clonedInsn = cgFunc->GetTheCFG()->CloneInsn(*insn); clonedInsn->SetPrev(nullptr); clonedInsn->SetNext(nullptr); @@ -822,10 +1053,334 @@ bool DuplicateBBPattern::Optimize(BB &curBB) return false; } +uint32 CrossJumpBBPattern::CheckBBInsnsMatch(BB &bb1, BB &bb2, Insn *&f1, Insn *&f2) +{ + // skip jump insn, all jump insn is compared in CheckBBSuccMatch + auto skipBBJumpInsn = [](BB &bb) { + auto *insn = bb.GetLastMachineInsn(); + if (insn != nullptr && insn->IsBranch()) { + insn = insn->GetPreviousMachineInsn(); + } + return insn; + }; + auto *insn1 = skipBBJumpInsn(bb1); + auto *insn2 = skipBBJumpInsn(bb2); + + uint32 ninsns = 0; + Insn *last1 = nullptr; + Insn *last2 = nullptr; + while (insn1 != nullptr && insn2 != nullptr) { + if (!insn1->Equals(*insn2)) { + break; + } + ++ninsns; + last1 = insn1; + last2 = insn2; + insn1 = insn1->GetPreviousMachineInsn(); + insn2 = insn2->GetPreviousMachineInsn(); + } + if (ninsns != 0) { + f1 = last1; + f2 = last2; + } + return ninsns; +} + +bool CrossJumpBBPattern::CheckBBSuccMatch(BB &bb1, BB &bb2) +{ + auto *brInsn1 = bb1.GetLastMachineInsn(); + auto *brInsn2 = bb2.GetLastMachineInsn(); + if (brInsn1 == nullptr || brInsn2 == nullptr) { + return false; + } + + // When both bb1 and bb2 have only one successor, and bb has no jump instruction or + // the jump instruction is a goto instruction, we can merge them directly. + if (bb1.GetSuccsSize() == 1 && + (!brInsn1->IsBranch() || cgFunc->GetTheCFG()->GetInsnModifier()->IsSimpleJumpInsn(*brInsn1))) { + return (bb2.GetSuccsSize() == 1 && + (!brInsn2->IsBranch() || cgFunc->GetTheCFG()->GetInsnModifier()->IsSimpleJumpInsn(*brInsn2))); + } + + // check conditional jumps + if (bb1.GetSuccsSize() == BB::kBBIfSuccsSize && brInsn1->IsCondBranch() && + bb2.GetSuccsSize() == BB::kBBIfSuccsSize && brInsn2->IsCondBranch()) { + auto *b1 = cgFunc->GetTheCFG()->GetTargetSuc(bb1); + auto *b2 = cgFunc->GetTheCFG()->GetTargetSuc(bb2); + auto *f1 = bb1.GetNext(); + auto *f2 = bb2.GetNext(); + + auto getSoloGotoTrueTarget = [this](BB &bb) { + return (bb.IsSoloGoto() ? cgFunc->GetTheCFG()->GetTargetSuc(bb) : &bb); + }; + f1 = getSoloGotoTrueTarget(*f1); + f2 = getSoloGotoTrueTarget(*f2); + + bool reverse = false; + if (f1 == f2 && b1 == b2) { + reverse = false; + } else if (f1 == b2 && b1 == f2) { + reverse = true; + } else { + return false; + } + + if (!reverse) { + return brInsn1->Equals(*brInsn2); + } + + auto newCode = FlipConditionOp(brInsn2->GetMachineOpcode()); + if (newCode != brInsn1->GetMachineOpcode()) { + return false; + } + auto targetIdx = GetJumpTargetIdx(*brInsn2); + for (uint32 i = 0; i < brInsn1->GetOperandSize(); ++i) { + if (i == targetIdx) { + continue; + } + if (!brInsn1->GetOperand(i).Equals(brInsn2->GetOperand(i))) { + return false; + } + } + auto &tgtOpnd1 = brInsn1->GetOperand(targetIdx); + auto &tgtOpnd2 = cgFunc->GetOrCreateLabelOperand(*f2); + return tgtOpnd1.Equals(tgtOpnd2); + } + return false; +} + +// split srcBB at lastInsn, and set newBB is srcBB's fallthru +BB &CrossJumpBBPattern::SplitBB(BB &srcBB, Insn &lastInsn) +{ + auto &newBB = *cgFunc->CreateNewBB(false, srcBB.GetKind(), srcBB.GetFrequency()); + + std::vector splitInsns; + for (auto *newInsn = lastInsn.GetNext(); newInsn != nullptr; newInsn = newInsn->GetNext()) { + splitInsns.push_back(newInsn); + } + srcBB.RemoveInsnSequence(*lastInsn.GetNext(), *srcBB.GetLastInsn()); + for (auto *insn : splitInsns) { + newBB.AppendInsn(*insn); + } + + for (auto *succ : srcBB.GetSuccs()) { + newBB.PushBackSuccs(*succ, srcBB.GetEdgeProb(*succ)); + succ->RemovePreds(srcBB); + succ->PushBackPreds(newBB); + } + srcBB.ClearSuccs(); + srcBB.PushBackSuccs(newBB); + newBB.PushBackPreds(srcBB); + + srcBB.SetKind(BB::kBBFallthru); + newBB.SetNext(srcBB.GetNext()); + srcBB.GetNext()->SetPrev(&newBB); + srcBB.SetNext(&newBB); + newBB.SetPrev(&srcBB); + return newBB; +} + +void CrossJumpBBPattern::GetMergeDirection(BB &bb1, BB &bb2, const Insn &f1, const Insn &f2, MergeDirection &dir) +{ + auto mergeDirection = [](MergeDirection dir1, MergeDirection dir2) { + if (dir1 == dir2) { + return dir1; + } + if (dir1 == kDirectionBoth) { + return dir2; + } else if (dir2 == kDirectionBoth) { + return dir1; + } + return kDirectionNone; + }; + + auto *brInsn1 = bb1.GetLastMachineInsn(); + auto *brInsn2 = bb2.GetLastMachineInsn(); + ASSERT_NOT_NULL(brInsn1); + ASSERT_NOT_NULL(brInsn2); + // check fallthru BB + if (!brInsn1->IsBranch() && brInsn2->IsBranch()) { // need merge backward + dir = mergeDirection(dir, kDirectionBackward); + } else if (brInsn1->IsBranch() && !brInsn2->IsBranch()) { // need merge forward + dir = mergeDirection(dir, kDirectionForward); + } else { // all is branch + auto checkAllMatchAndIsJumpTarget = [](const BB &bb, const Insn &f) { + if (&f != bb.GetFirstMachineInsn()) { + return false; + } + for (auto *pred : bb.GetPreds()) { + if (pred->GetNext() == &bb) { + return false; + } + } + return true; + }; + bool c1 = checkAllMatchAndIsJumpTarget(bb1, f1); + bool c2 = checkAllMatchAndIsJumpTarget(bb2, f2); + // When a BB's all instructions match and is the jump target, after this optimization, + // it will become a solo goto BB and is deleted in other optimizations. + if (c1 && c2) { + dir = mergeDirection(dir, kDirectionBoth); + } else if (c1) { + dir = mergeDirection(dir, kDirectionForward); + } else if (c2) { + dir = mergeDirection(dir, kDirectionBackward); + } else if (!CGOptions::GetInstance().OptimizeForSize()) { + // Two BBs are partially matched. If CrossJump optimization is performed, a jump + // instruction is introduced. Therefore, when the optimization is not for size, + // this optimization is not performed. + dir = kDirectionNone; + } else { + dir = mergeDirection(dir, kDirectionBoth); + } + } +} + +void CrossJumpBBPattern::MergeMemInfo(BB &bb1, Insn &newpos1, BB &redirectBB) +{ + CHECK_FATAL(newpos1.GetBB() == &bb1, "NIY, insn must in bb"); + auto *insn1 = &newpos1; + auto *insn2 = redirectBB.GetFirstMachineInsn(); + while (insn1 != nullptr && insn2 != nullptr && !insn1->IsBranch() && !insn2->IsBranch()) { + DEBUG_ASSERT(insn1->Equals(*insn2), "NIY, insn must equal"); + insn2->MergeReferenceOsts(*insn1); + insn1 = insn1->GetNextMachineInsn(); + insn2 = insn2->GetNextMachineInsn(); + } +} + +// try cross jump opt +bool CrossJumpBBPattern::TryCrossJumpBB(BB &bb1, BB &bb2, MergeDirection dir) +{ + // if bb is solo goto, we need to get its actual precursor bb. + auto skipSoloGotoBB = [](BB &bb) { + if (bb.IsSoloGoto() && bb.GetPredsSize() == 1) { + return bb.GetPreds().front(); + } + return &bb; + }; + auto *src1 = skipSoloGotoBB(bb1); + auto *src2 = skipSoloGotoBB(bb2); + if (src1 == cgFunc->GetFirstBB() || src2 == cgFunc->GetFirstBB() || src1 == src2) { + return false; + } + if (src1->IsUnreachable() || src2->IsUnreachable()) { + return false; + } + + if (!CheckBBSuccMatch(*src1, *src2)) { + return false; + } + + Insn *newpos1 = nullptr; // position in src1 that will be split + Insn *newpos2 = nullptr; // position in src2 that will be split + auto nmatch = CheckBBInsnsMatch(*src1, *src2, newpos1, newpos2); + if (nmatch == 0 && (newpos1 == nullptr || newpos1 != src1->GetFirstMachineInsn())) { + return false; + } + GetMergeDirection(bb1, bb2, *newpos1, *newpos2, dir); + if (dir == kDirectionNone) { + return false; + } else if (dir == kDirectionBackward) { // backward merge, swap them + std::swap(src1, src2); + std::swap(newpos1, newpos2); + } + + auto *redirectBB = src2; + if (newpos2 != src2->GetFirstMachineInsn()) { + redirectBB = &SplitBB(*src2, *newpos2->GetPreviousMachineInsn()); + } + MergeMemInfo(*src1, *newpos1, *redirectBB); + src1->RemoveInsnSequence(*newpos1, *src1->GetLastInsn()); + auto &brTarget = cgFunc->GetOrCreateLabelOperand(*redirectBB); + auto &uncondInsn = cgFunc->GetInsnBuilder()->BuildInsn(GetUnCondBranchMOP(), brTarget); + src1->AppendInsn(uncondInsn); + src1->SetKind(BB::kBBGoto); + for (auto *succ : src1->GetSuccs()) { + succ->RemovePreds(*src1); + } + src1->ClearSuccs(); + src1->PushBackSuccs(*redirectBB); + redirectBB->PushBackPreds(*src1); + + return true; +} + +bool CrossJumpBBPattern::OptimizeOnce(const BB &curBB) +{ + // Merge otherBB to fallthruBB is beneficial in most case, as that adds no + // branches to the program. We'll try that combination first. + auto *fallthru = FindPredFallthruBB(curBB); + for (auto it1 = curBB.GetPreds().begin(); it1 != curBB.GetPreds().end(); ++it1) { + auto *pred = *it1; + if (fallthru != nullptr) { + if (pred == fallthru) { + continue; + } + + if (TryCrossJumpBB(*pred, *fallthru, kDirectionForward)) { + return true; // Cross jump opt may modify preds or fallthruBB, so we can't continue. + } + } + + for (auto it2 = it1; it2 != curBB.GetPreds().end(); ++it2) { + auto *pred2 = *it2; + if (pred == pred2 || pred2 == fallthru) { + continue; + } + if (TryCrossJumpBB(*pred, *pred2, kDirectionBoth)) { + return true; // Cross jump opt may modify preds or fallthruBB, so we can't continue. + } + } + } + return false; +} + +bool CrossJumpBBPattern::Optimize(BB &curBB) +{ + if (!cgFunc->IsAfterRegAlloc() || curBB.IsUnreachable()) { + return false; + } + if (curBB.GetPredsSize() < kMinCrossJumpPreds || curBB.GetPredsSize() >= kMaxCrossJumpPreds) { + return false; + } + + bool cfgChanged = false; + bool optimized = false; + do { + cfgChanged = false; + cfgChanged |= OptimizeOnce(curBB); + optimized |= cfgChanged; + } while (cfgChanged); + + return optimized; +} + /* === new pm === */ +bool CgPreCfgo::PhaseRun(maplebe::CGFunc &f) +{ + CFGOptimizer *cfgOptimizer = f.GetCG()->CreateCFGOptimizer(*GetPhaseMemPool(), f); + const std::string &funcClass = f.GetFunction().GetBaseClassName(); + const std::string &funcName = f.GetFunction().GetBaseFuncName(); + const std::string &name = funcClass + funcName; + if (CFGO_DUMP_NEWPM) { + DotGenerator::GenerateDot("before-precfgo", f, f.GetMirModule()); + } + cfgOptimizer->Run(name); + if (CFGO_DUMP_NEWPM) { + f.GetTheCFG()->CheckCFG(); + DotGenerator::GenerateDot("after-precfgo", f, f.GetMirModule()); + } + return false; +} +MAPLE_TRANSFORM_PHASE_REGISTER_CANSKIP(CgPreCfgo, precfgo) + bool CgCfgo::PhaseRun(maplebe::CGFunc &f) { CFGOptimizer *cfgOptimizer = f.GetCG()->CreateCFGOptimizer(*GetPhaseMemPool(), f); + if (f.IsAfterRegAlloc()) { + cfgOptimizer->SetPhase(kCfgoPostRegAlloc); + } const std::string &funcClass = f.GetFunction().GetBaseClassName(); const std::string &funcName = f.GetFunction().GetBaseFuncName(); const std::string &name = funcClass + funcName; @@ -834,6 +1389,7 @@ bool CgCfgo::PhaseRun(maplebe::CGFunc &f) } cfgOptimizer->Run(name); if (CFGO_DUMP_NEWPM) { + f.GetTheCFG()->CheckCFG(); DotGenerator::GenerateDot("after-cfgo", f, f.GetMirModule()); } return false; @@ -846,6 +1402,7 @@ bool CgPostCfgo::PhaseRun(maplebe::CGFunc &f) const std::string &funcClass = f.GetFunction().GetBaseClassName(); const std::string &funcName = f.GetFunction().GetBaseFuncName(); const std::string &name = funcClass + funcName; + cfgOptimizer->SetPhase(kPostCfgo); if (CFGO_DUMP_NEWPM) { DotGenerator::GenerateDot("before-postcfgo", f, f.GetMirModule()); } diff --git a/ecmascript/compiler/codegen/maple/maple_be/src/cg/cg_cfg.cpp b/ecmascript/compiler/codegen/maple/maple_be/src/cg/cg_cfg.cpp index 668f5a27cfdde50f48e168cd02b8af49e3b09dec..9d71fff5f53f68b3d2784ef329dcae302fb2c922 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/src/cg/cg_cfg.cpp +++ b/ecmascript/compiler/codegen/maple/maple_be/src/cg/cg_cfg.cpp @@ -34,7 +34,8 @@ namespace { using namespace maplebe; bool CanBBThrow(const BB &bb) { - FOR_BB_INSNS_CONST(insn, &bb) { + FOR_BB_INSNS_CONST(insn, &bb) + { if (insn->IsTargetInsn() && insn->CanThrow()) { return true; } @@ -160,40 +161,46 @@ void CGCFG::BuildCFG() void CGCFG::CheckCFG() { - FOR_ALL_BB(bb, cgFunc) { + FOR_ALL_BB(bb, cgFunc) + { for (BB *sucBB : bb->GetSuccs()) { bool found = false; for (BB *sucPred : sucBB->GetPreds()) { if (sucPred == bb) { - if (found == false) { + if (!found) { found = true; } else { LogInfo::MapleLogger() << "dup pred " << sucPred->GetId() << " for sucBB " << sucBB->GetId() << "\n"; + CHECK_FATAL_FALSE("CG_CFG check failed !"); } } } - if (found == false) { + if (!found) { LogInfo::MapleLogger() << "non pred for sucBB " << sucBB->GetId() << " for BB " << bb->GetId() << "\n"; + CHECK_FATAL_FALSE("CG_CFG check failed !"); } } } - FOR_ALL_BB(bb, cgFunc) { + FOR_ALL_BB(bb, cgFunc) + { for (BB *predBB : bb->GetPreds()) { bool found = false; for (BB *predSucc : predBB->GetSuccs()) { if (predSucc == bb) { - if (found == false) { + if (!found) { found = true; } else { LogInfo::MapleLogger() << "dup succ " << predSucc->GetId() << " for predBB " << predBB->GetId() << "\n"; + CHECK_FATAL_FALSE("CG_CFG check failed !"); } } } - if (found == false) { + if (!found) { LogInfo::MapleLogger() << "non succ for predBB " << predBB->GetId() << " for BB " << bb->GetId() << "\n"; + CHECK_FATAL_FALSE("CG_CFG check failed !"); } } } @@ -213,7 +220,8 @@ void CGCFG::CheckCFGFreq() CHECK_FATAL(false, "Verifyfreq failure BB frequency!"); } }; - FOR_ALL_BB(bb, cgFunc) { + FOR_ALL_BB(bb, cgFunc) + { if (bb->IsUnreachable() || bb->IsCleanup()) { continue; } @@ -555,6 +563,24 @@ void CGCFG::RemoveBB(BB &curBB, bool isGotoIf) } } +void CGCFG::UpdateCommonExitBBInfo() +{ + BB *commonExitBB = cgFunc->GetCommonExitBB(); + ASSERT_NOT_NULL(commonExitBB); + commonExitBB->ClearPreds(); + for (BB *exitBB : cgFunc->GetExitBBsVec()) { + if (!exitBB->IsUnreachable()) { + commonExitBB->PushBackPreds(*exitBB); + } + } + for (BB *noRetBB : cgFunc->GetNoRetCallBBVec()) { + if (!noRetBB->IsUnreachable()) { + commonExitBB->PushBackPreds(*noRetBB); + } + } + WontExitAnalysis(); +} + void CGCFG::RetargetJump(BB &srcBB, BB &targetBB) { insnVisitor->ModifyJumpTarget(srcBB, targetBB); @@ -607,16 +633,25 @@ BB *CGCFG::GetTargetSuc(BB &curBB, bool branchOnly, bool isGotoIf) return nullptr; } -bool CGCFG::InLSDA(LabelIdx label, const EHFunc &ehFunc) +bool CGCFG::InLSDA(LabelIdx label [[maybe_unused]], const EHFunc *ehFunc [[maybe_unused]]) { - if (!label || ehFunc.GetLSDACallSiteTable() == nullptr) { +#ifndef ONLY_C + /* the function have no exception handle module */ + if (ehFunc == nullptr) { return false; } - if (label == ehFunc.GetLSDACallSiteTable()->GetCSTable().GetEndOffset()->GetLabelIdx() || - label == ehFunc.GetLSDACallSiteTable()->GetCSTable().GetStartOffset()->GetLabelIdx()) { + + if ((label == 0) || ehFunc->GetLSDACallSiteTable() == nullptr) { + return false; + } + if (label == ehFunc->GetLSDACallSiteTable()->GetCSTable().GetEndOffset()->GetLabelIdx() || + label == ehFunc->GetLSDACallSiteTable()->GetCSTable().GetStartOffset()->GetLabelIdx()) { return true; } - return ehFunc.GetLSDACallSiteTable()->InCallSiteTable(label); + return ehFunc->GetLSDACallSiteTable()->InCallSiteTable(label); +#else + return false; +#endif } bool CGCFG::InSwitchTable(LabelIdx label, const CGFunc &func) @@ -632,6 +667,11 @@ bool CGCFG::IsCompareAndBranchInsn(const Insn &insn) const return insnVisitor->IsCompareAndBranchInsn(insn); } +bool CGCFG::IsTestAndBranchInsn(const Insn &insn) const +{ + return insnVisitor->IsTestAndBranchInsn(insn); +} + bool CGCFG::IsAddOrSubInsn(const Insn &insn) const { return insnVisitor->IsAddOrSubInsn(insn); @@ -642,7 +682,8 @@ Insn *CGCFG::FindLastCondBrInsn(BB &bb) const if (bb.GetKind() != BB::kBBIf) { return nullptr; } - FOR_BB_INSNS_REV(insn, (&bb)) { + FOR_BB_INSNS_REV(insn, (&bb)) + { if (insn->IsBranch()) { return insn; } @@ -731,7 +772,7 @@ void CGCFG::UnreachCodeAnalysis() EHFunc *ehFunc = cgFunc->GetEHFunc(); /* if unreachBB InLSDA ,replace unreachBB's label with nextReachableBB before remove it. */ if (ehFunc != nullptr && ehFunc->NeedFullLSDA() && - cgFunc->GetTheCFG()->InLSDA(unreachBB->GetLabIdx(), *ehFunc)) { + cgFunc->GetTheCFG()->InLSDA(unreachBB->GetLabIdx(), ehFunc)) { /* find next reachable BB */ BB *nextReachableBB = nullptr; for (BB *curBB = unreachBB; curBB != nullptr; curBB = curBB->GetNext()) { @@ -808,7 +849,8 @@ void CGCFG::WontExitAnalysis() BB *CGCFG::FindLastRetBB() { - FOR_ALL_BB_REV(bb, cgFunc) { + FOR_ALL_BB_REV(bb, cgFunc) + { if (bb->GetKind() == BB::kBBReturn) { return bb; } diff --git a/ecmascript/compiler/codegen/maple/maple_be/src/cg/cg_ssa.cpp b/ecmascript/compiler/codegen/maple/maple_be/src/cg/cg_ssa.cpp index d543af21e733b9c0212da096c7decc50b9b94f65..f9aff79c66ead125f326cf7cb7d28f5b69d0e049 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/src/cg/cg_ssa.cpp +++ b/ecmascript/compiler/codegen/maple/maple_be/src/cg/cg_ssa.cpp @@ -27,8 +27,10 @@ void CGSSAInfo::ConstructSSA() RenameVariablesForBB(domInfo->GetCommonEntryBB().GetId()); #if DEBUG /* Check phiListOpnd, must be ssaForm */ - FOR_ALL_BB(bb, cgFunc) { - FOR_BB_INSNS(insn, bb) { + FOR_ALL_BB(bb, cgFunc) + { + FOR_BB_INSNS(insn, bb) + { if (!insn->IsPhi()) { continue; } @@ -57,8 +59,10 @@ void CGSSAInfo::MarkInsnsInSSA(Insn &insn) void CGSSAInfo::InsertPhiInsn() { - FOR_ALL_BB(bb, cgFunc) { - FOR_BB_INSNS(insn, bb) { + FOR_ALL_BB(bb, cgFunc) + { + FOR_BB_INSNS(insn, bb) + { if (!insn->IsMachineInstruction()) { continue; } @@ -98,7 +102,8 @@ void CGSSAInfo::PrunedPhiInsertion(const BB &bb, RegOperand &virtualOpnd) Insn &phiInsn = codeGen->BuildPhiInsn(virtualOpnd, phiList); MarkInsnsInSSA(phiInsn); bool insertSuccess = false; - FOR_BB_INSNS(insn, phiBB) { + FOR_BB_INSNS(insn, phiBB) + { if (insn->IsMachineInstruction()) { (void)phiBB->InsertInsnBefore(*insn, phiInsn); insertSuccess = true; @@ -138,7 +143,8 @@ void CGSSAInfo::RenameBB(BB &bb) oriStackSize[it.first] = static_cast(it.second.size()); } RenamePhi(bb); - FOR_BB_INSNS(insn, &bb) { + FOR_BB_INSNS(insn, &bb) + { if (!insn->IsMachineInstruction()) { continue; } @@ -258,10 +264,24 @@ void CGSSAInfo::SetReversePostOrder() } } +Insn *CGSSAInfo::GetDefInsn(const RegOperand &useReg) +{ + if (!useReg.IsSSAForm()) { + return nullptr; + } + regno_t useRegNO = useReg.GetRegisterNumber(); + VRegVersion *useVersion = FindSSAVersion(useRegNO); + CHECK_FATAL(useVersion != nullptr, "useVRegVersion must not be null based on ssa"); + CHECK_FATAL(!useVersion->IsDeleted(), "deleted version"); + DUInsnInfo *defInfo = useVersion->GetDefInsnInfo(); + return defInfo == nullptr ? nullptr : defInfo->GetInsn(); +} + void CGSSAInfo::DumpFuncCGIRinSSAForm() const { LogInfo::MapleLogger() << "\n****** SSA CGIR for " << cgFunc->GetName() << " *******\n"; - FOR_ALL_BB_CONST(bb, cgFunc) { + FOR_ALL_BB_CONST(bb, cgFunc) + { LogInfo::MapleLogger() << "=== BB " << " <" << bb->GetKindName(); if (bb->GetLabIdx() != MIRLabelTable::GetDummyLabel()) { @@ -294,7 +314,8 @@ void CGSSAInfo::DumpFuncCGIRinSSAForm() const LogInfo::MapleLogger() << "===\n"; LogInfo::MapleLogger() << "frequency:" << bb->GetFrequency() << "\n"; - FOR_BB_INSNS_CONST(insn, bb) { + FOR_BB_INSNS_CONST(insn, bb) + { if (insn->IsCfiInsn() && insn->IsDbgInsn()) { insn->Dump(); } else { diff --git a/ecmascript/compiler/codegen/maple/maple_be/src/cg/cgfunc.cpp b/ecmascript/compiler/codegen/maple/maple_be/src/cg/cgfunc.cpp index d06d4c3512b0adde69e52e6e6b721ec5486ce610..2708fa0a4001954a79e2b414cc7c22276faeae78 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/src/cg/cgfunc.cpp +++ b/ecmascript/compiler/codegen/maple/maple_be/src/cg/cgfunc.cpp @@ -1635,6 +1635,7 @@ CGFunc::CGFunc(MIRModule &mod, CG &cg, MIRFunction &mirFunc, BECommon &beCommon, stackMp(stackMp), func(mirFunc), exitBBVec(allocator.Adapter()), + noReturnCallBBVec(allocator.Adapter()), extendSet(allocator.Adapter()), lab2BBMap(allocator.Adapter()), beCommon(beCommon), @@ -2074,12 +2075,14 @@ void CGFunc::TraverseAndClearCatchMark(BB &bb) void CGFunc::MarkCatchBBs() { /* First, suspect all bb to be in catch */ - FOR_ALL_BB(bb, this) { + FOR_ALL_BB(bb, this) + { bb->SetIsCatch(true); bb->SetInternalFlag3(0); /* mark as not visited */ } /* Eliminate cleanup section from catch */ - FOR_ALL_BB(bb, this) { + FOR_ALL_BB(bb, this) + { if (bb->GetFirstStmt() == cleanupLabel) { bb->SetIsCatch(false); DEBUG_ASSERT(bb->GetSuccs().size() <= 1, "MarkCatchBBs incorrect cleanup label"); @@ -2113,7 +2116,8 @@ void CGFunc::MarkCatchBBs() void CGFunc::MarkCleanupEntryBB() { BB *cleanupEntry = nullptr; - FOR_ALL_BB(bb, this) { + FOR_ALL_BB(bb, this) + { bb->SetIsCleanup(0); /* Use to mark cleanup bb */ bb->SetInternalFlag3(0); /* Use to mark if visited. */ if (bb->GetFirstStmt() == this->cleanupLabel) { @@ -2134,7 +2138,8 @@ void CGFunc::MarkCleanupEntryBB() /* Check if all of the cleanup bb is at bottom of the function. */ bool isCleanupArea = true; if (!mirModule.IsCModule()) { - FOR_ALL_BB_REV(bb, this) { + FOR_ALL_BB_REV(bb, this) + { if (isCleanupArea) { DEBUG_ASSERT(bb->IsCleanup(), "CG internal error, cleanup BBs should be at the bottom of the function."); @@ -2286,7 +2291,8 @@ void CGFunc::UpdateCallBBFrequency() if (!func.HasFreqMap() || func.GetLastFreqMap().empty()) { return; } - FOR_ALL_BB(bb, this) { + FOR_ALL_BB(bb, this) + { if (bb->GetKind() != BB::kBBFallthru || !bb->HasCall()) { continue; } @@ -2363,7 +2369,8 @@ void CGFunc::DumpCFG() const MIRSymbol *funcSt = GlobalTables::GetGsymTable().GetSymbolFromStidx(func.GetStIdx().Idx()); DEBUG_ASSERT(funcSt != nullptr, "null ptr check"); LogInfo::MapleLogger() << "\n****** CFG built by CG for " << funcSt->GetName() << " *******\n"; - FOR_ALL_BB_CONST(bb, this) { + FOR_ALL_BB_CONST(bb, this) + { LogInfo::MapleLogger() << "=== BB ( " << std::hex << bb << std::dec << " ) <" << bb->GetKindName() << "> ===\n"; LogInfo::MapleLogger() << "BB id:" << bb->GetId() << "\n"; if (!bb->GetPreds().empty()) { @@ -2400,7 +2407,8 @@ void CGFunc::DumpCGIR() const MIRSymbol *funcSt = GlobalTables::GetGsymTable().GetSymbolFromStidx(func.GetStIdx().Idx()); DEBUG_ASSERT(funcSt != nullptr, "null ptr check"); LogInfo::MapleLogger() << "\n****** CGIR for " << funcSt->GetName() << " *******\n"; - FOR_ALL_BB_CONST(bb, this) { + FOR_ALL_BB_CONST(bb, this) + { if (bb->IsUnreachable()) { continue; } @@ -2452,7 +2460,8 @@ void CGFunc::DumpCGIR() const LogInfo::MapleLogger() << "===\n"; LogInfo::MapleLogger() << "frequency:" << bb->GetFrequency() << "\n"; - FOR_BB_INSNS_CONST(insn, bb) { + FOR_BB_INSNS_CONST(insn, bb) + { insn->Dump(); } } @@ -2469,7 +2478,8 @@ void CGFunc::ClearLoopInfo() { loops.clear(); loops.shrink_to_fit(); - FOR_ALL_BB(bb, this) { + FOR_ALL_BB(bb, this) + { bb->ClearLoopPreds(); bb->ClearLoopSuccs(); } diff --git a/ecmascript/compiler/codegen/maple/maple_be/src/cg/insn.cpp b/ecmascript/compiler/codegen/maple/maple_be/src/cg/insn.cpp index 9d9f4555d0cc013728ec8fb0d485f34107543aa1..f0f846395a14a2dc57193fd50b6372d96ac3701e 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/src/cg/insn.cpp +++ b/ecmascript/compiler/codegen/maple/maple_be/src/cg/insn.cpp @@ -81,6 +81,10 @@ bool Insn::IsCall() const DEBUG_ASSERT(md, " set insnDescription for insn "); return md->IsCall(); } +bool Insn::IsSpecialCall() const +{ + return md ? md->IsSpecialCall() : false; +} bool Insn::IsTailCall() const { DEBUG_ASSERT(md, " set insnDescription for insn "); @@ -204,6 +208,19 @@ Operand *Insn::GetMemOpnd() const } return nullptr; } + +uint32 Insn::GetMemOpndIdx() const +{ + uint32 opndIdx = kInsnMaxOpnd; + for (uint32 i = 0; i < static_cast(opnds.size()); ++i) { + Operand &opnd = GetOperand(i); + if (opnd.IsMemoryAccessOperand()) { + return i; + } + } + return opndIdx; +} + void Insn::SetMemOpnd(MemOperand *memOpnd) { for (uint32 i = 0; i < static_cast(opnds.size()); ++i) { diff --git a/ecmascript/compiler/codegen/maple/maple_be/src/cg/loop.cpp b/ecmascript/compiler/codegen/maple/maple_be/src/cg/loop.cpp index 99ad0ab37ea90c4406ed3268146e5dc26e59b63d..f713f331e4033e027e702d64f3b08e643be7423c 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/src/cg/loop.cpp +++ b/ecmascript/compiler/codegen/maple/maple_be/src/cg/loop.cpp @@ -69,6 +69,11 @@ void LoopHierarchy::PrintLoops(const std::string &name) const } } +bool CGFuncLoops::IsBBLoopMember(const BB *bb) +{ + return (*(std::find(loopMembers.begin(), loopMembers.end(), bb)) == bb); +} + void CGFuncLoops::CheckOverlappingInnerLoops(const MapleVector &iLoops, const MapleVector &loopMem) const { @@ -307,7 +312,8 @@ void LoopFinder::markExtraEntryAndEncl() // Collect all entries bool foundNewEntry = false; fill(visitedBBs.begin(), visitedBBs.end(), false); - FOR_ALL_BB(bb, cgFunc) { + FOR_ALL_BB(bb, cgFunc) + { if (!visitedBBs[bb->GetId()]) { dfsBBs.push(bb); visitedBBs[bb->GetId()] = true; @@ -639,13 +645,15 @@ void LoopFinder::FormLoopHierarchy() onPathBBs.clear(); onPathBBs.resize(cgFunc->NumBBs(), false); - FOR_ALL_BB(bb, cgFunc) { + FOR_ALL_BB(bb, cgFunc) + { bb->SetLevel(0); } bool changed; do { changed = false; - FOR_ALL_BB(bb, cgFunc) { + FOR_ALL_BB(bb, cgFunc) + { if (!visitedBBs[bb->GetId()]) { dfsBBs.push(bb); seekCycles(); diff --git a/ecmascript/compiler/codegen/maple/maple_be/src/cg/optimize_common.cpp b/ecmascript/compiler/codegen/maple/maple_be/src/cg/optimize_common.cpp index c0e9d7b2972b11c319d1bbf229a35d8d1ded5858..2691381e98dc3ce9415bc2339b9ec1ada2b6b4dd 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/src/cg/optimize_common.cpp +++ b/ecmascript/compiler/codegen/maple/maple_be/src/cg/optimize_common.cpp @@ -33,25 +33,30 @@ void Optimizer::Run(const std::string &funcName, bool checkOnly) } /* Search the cgFunc for multiple possible optimizations in one pass */ if (!singlePassPatterns.empty()) { - BB *curBB = cgFunc->GetFirstBB(); - bool flag = false; - while (curBB != nullptr) { + bool changed = false; + do { + changed = false; for (OptimizationPattern *p : singlePassPatterns) { - if (p->Optimize(*curBB)) { - flag = p->IsKeepPosition(); - p->SetKeepPosition(false); - break; + BB *curBB = cgFunc->GetFirstBB(); + while (curBB != nullptr) { + if (p->Optimize(*curBB)) { + changed = true; + } + if (p->IsKeepPosition()) { + p->SetKeepPosition(false); + } else { + curBB = curBB->GetNext(); + } } } - - if (flag) { - flag = false; - } else { - curBB = curBB->GetNext(); - } - } + } while (changed); } + // Update commonExitBB info, especially in infinite loop case. + // But we can not get the commonExitBB by traversal CFG, so + // it needs to be handled separately. + cgFunc->GetTheCFG()->UpdateCommonExitBBInfo(); + if (CGOptions::IsDumpOptimizeCommonLog()) { constexpr int arrSize = 80; char post[arrSize]; @@ -64,14 +69,17 @@ void Optimizer::Run(const std::string &funcName, bool checkOnly) OptimizeLogger::GetLogger().ClearLocal(); } -void OptimizationPattern::Search2Op(bool noOptimize) +bool OptimizationPattern::Search2Op(bool noOptimize) { + bool hasChange = false; checkOnly = noOptimize; + InitPattern(); BB *curBB = cgFunc->GetFirstBB(); while (curBB != nullptr) { bool changed = false; do { changed = Optimize(*curBB); + hasChange = hasChange || changed; } while (changed); if (keepPosition) { keepPosition = false; @@ -79,6 +87,7 @@ void OptimizationPattern::Search2Op(bool noOptimize) curBB = curBB->GetNext(); } } + return hasChange; } void OptimizationPattern::Log(uint32 bbID) @@ -141,7 +150,8 @@ bool DotGenerator::IsBackEdge(const CGFunc &cgFunction, const BB &from, const BB void DotGenerator::DumpEdge(const CGFunc &cgFunction, std::ofstream &cfgFileOfStream, bool isIncludeEH) { - FOR_ALL_BB_CONST(bb, &cgFunction) { + FOR_ALL_BB_CONST(bb, &cgFunction) + { for (auto *succBB : bb->GetSuccs()) { cfgFileOfStream << "BB" << bb->GetId(); cfgFileOfStream << " -> " @@ -212,9 +222,11 @@ bool DotGenerator::FoundNormalOpndRegNum(const RegOperand ®Opnd, const Insn & void DotGenerator::DumpBBInstructions(const CGFunc &cgFunction, regno_t vReg, std::ofstream &cfgFile) { - FOR_ALL_BB_CONST(bb, &cgFunction) { + FOR_ALL_BB_CONST(bb, &cgFunction) + { if (vReg != 0) { - FOR_BB_INSNS_CONST(insn, bb) { + FOR_BB_INSNS_CONST(insn, bb) + { bool found = false; uint32 opndNum = insn->GetOperandSize(); for (uint32 i = 0; i < opndNum; ++i) { diff --git a/ecmascript/compiler/codegen/maple/maple_be/src/cg/peep.cpp b/ecmascript/compiler/codegen/maple/maple_be/src/cg/peep.cpp index 59784e05752f82b6d9b929c88391a93c26ec2c43..64dfb84c7842f1c10baf062e548d5a0b3f42578a 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/src/cg/peep.cpp +++ b/ecmascript/compiler/codegen/maple/maple_be/src/cg/peep.cpp @@ -30,7 +30,7 @@ namespace maplebe { #if TARGAARCH64 -bool CGPeepPattern::IsCCRegCrossVersion(Insn &startInsn, Insn &endInsn, const RegOperand &ccReg) +bool CGPeepPattern::IsCCRegCrossVersion(Insn &startInsn, Insn &endInsn, const RegOperand &ccReg) const { if (startInsn.GetBB() != endInsn.GetBB()) { return true; @@ -74,7 +74,7 @@ int64 CGPeepPattern::GetLogValueAtBase2(int64 val) const return (__builtin_popcountll(static_cast(val)) == 1) ? (__builtin_ffsll(val) - 1) : -1; } -InsnSet CGPeepPattern::GetAllUseInsn(const RegOperand &defReg) +InsnSet CGPeepPattern::GetAllUseInsn(const RegOperand &defReg) const { InsnSet allUseInsn; if ((ssaInfo != nullptr) && defReg.IsSSAForm()) { @@ -170,7 +170,7 @@ bool CGPeepPattern::IfOperandIsLiveAfterInsn(const RegOperand ®Opnd, Insn &in } } } else if (opnd.IsList()) { - auto &opndList = static_cast(opnd).GetOperands(); + auto &opndList = static_cast(opnd).GetOperands(); if (find(opndList.begin(), opndList.end(), ®Opnd) != opndList.end()) { return true; } @@ -184,7 +184,7 @@ bool CGPeepPattern::IfOperandIsLiveAfterInsn(const RegOperand ®Opnd, Insn &in continue; } const InsnDesc *md = nextInsn->GetDesc(); - auto *regProp = (md->opndMD[static_cast(i)]); + auto *regProp = (md->opndMD[static_cast(i)]); bool isUse = regProp->IsUse(); /* if noUse Redefined, no need to check live-out. */ return isUse; @@ -202,7 +202,8 @@ bool CGPeepPattern::FindRegLiveOut(const RegOperand ®Opnd, const BB &bb) * and the internal_flags3 should be cleared. */ if (PeepOptimizer::index == 0) { - FOR_ALL_BB(currbb, cgFunc) { + FOR_ALL_BB(currbb, cgFunc) + { currbb->SetInternalFlag3(0); } } @@ -285,7 +286,8 @@ bool CGPeepPattern::CheckRegLiveinReturnBB(const RegOperand ®Opnd, const BB & */ ReturnType CGPeepPattern::IsOpndLiveinBB(const RegOperand ®Opnd, const BB &bb) const { - FOR_BB_INSNS_CONST(insn, &bb) { + FOR_BB_INSNS_CONST(insn, &bb) + { if (!insn->IsMachineInstruction()) { continue; } @@ -293,7 +295,7 @@ ReturnType CGPeepPattern::IsOpndLiveinBB(const RegOperand ®Opnd, const BB &bb int32 lastOpndId = static_cast(insn->GetOperandSize() - 1); for (int32 i = lastOpndId; i >= 0; --i) { Operand &opnd = insn->GetOperand(static_cast(i)); - auto *regProp = (md->opndMD[static_cast(i)]); + auto *regProp = (md->opndMD[static_cast(i)]); if (opnd.IsConditionCode()) { if (regOpnd.GetRegisterNumber() == kRFLAG) { bool isUse = regProp->IsUse(); @@ -307,7 +309,7 @@ ReturnType CGPeepPattern::IsOpndLiveinBB(const RegOperand ®Opnd, const BB &bb auto &listOpnd = static_cast(opnd); if (insn->GetMachineOpcode() == MOP_asm) { if (static_cast(i) == kAsmOutputListOpnd || static_cast(i) == kAsmClobberListOpnd) { - for (auto op : listOpnd.GetOperands()) { + for (const auto op : listOpnd.GetOperands()) { if (op->GetRegisterNumber() == regOpnd.GetRegisterNumber()) { return kResDefFirst; } @@ -318,7 +320,7 @@ ReturnType CGPeepPattern::IsOpndLiveinBB(const RegOperand ®Opnd, const BB &bb } /* fall thru for kAsmInputListOpnd */ } - for (auto op : listOpnd.GetOperands()) { + for (const auto op : listOpnd.GetOperands()) { if (op->GetRegisterNumber() == regOpnd.GetRegisterNumber()) { return kResUseFirst; } @@ -357,7 +359,7 @@ ReturnType CGPeepPattern::IsOpndLiveinBB(const RegOperand ®Opnd, const BB &bb return kResNotFind; } -int PeepPattern::logValueAtBase2(int64 val) const +int PeepPattern::LogValueAtBase2(int64 val) const { return (__builtin_popcountll(static_cast(val)) == 1) ? (__builtin_ffsll(val) - 1) : (-1); } @@ -390,7 +392,7 @@ bool PeepPattern::IfOperandIsLiveAfterInsn(const RegOperand ®Opnd, Insn &insn } } } else if (opnd.IsList()) { - auto &opndList = static_cast(opnd).GetOperands(); + auto &opndList = static_cast(opnd).GetOperands(); if (find(opndList.begin(), opndList.end(), ®Opnd) != opndList.end()) { return true; } @@ -422,7 +424,8 @@ bool PeepPattern::FindRegLiveOut(const RegOperand ®Opnd, const BB &bb) * and the internal_flags3 should be cleared. */ if (PeepOptimizer::index == 0) { - FOR_ALL_BB(currbb, &cgFunc) { + FOR_ALL_BB(currbb, &cgFunc) + { currbb->SetInternalFlag3(0); } } @@ -505,7 +508,8 @@ bool PeepPattern::CheckRegLiveinReturnBB(const RegOperand ®Opnd, const BB &bb */ ReturnType PeepPattern::IsOpndLiveinBB(const RegOperand ®Opnd, const BB &bb) const { - FOR_BB_INSNS_CONST(insn, &bb) { + FOR_BB_INSNS_CONST(insn, &bb) + { if (!insn->IsMachineInstruction()) { continue; } @@ -513,7 +517,7 @@ ReturnType PeepPattern::IsOpndLiveinBB(const RegOperand ®Opnd, const BB &bb) int32 lastOpndId = static_cast(insn->GetOperandSize() - 1); for (int32 i = lastOpndId; i >= 0; --i) { Operand &opnd = insn->GetOperand(static_cast(i)); - auto *regProp = (md->opndMD[static_cast(i)]); + auto *regProp = (md->opndMD[static_cast(i)]); if (opnd.IsConditionCode()) { if (regOpnd.GetRegisterNumber() == kRFLAG) { bool isUse = regProp->IsUse(); @@ -527,7 +531,7 @@ ReturnType PeepPattern::IsOpndLiveinBB(const RegOperand ®Opnd, const BB &bb) auto &listOpnd = static_cast(opnd); if (insn->GetMachineOpcode() == MOP_asm) { if (static_cast(i) == kAsmOutputListOpnd || static_cast(i) == kAsmClobberListOpnd) { - for (auto op : listOpnd.GetOperands()) { + for (const auto op : listOpnd.GetOperands()) { if (op->GetRegisterNumber() == regOpnd.GetRegisterNumber()) { return kResDefFirst; } @@ -538,7 +542,7 @@ ReturnType PeepPattern::IsOpndLiveinBB(const RegOperand ®Opnd, const BB &bb) } /* fall thru for kAsmInputListOpnd */ } - for (auto op : listOpnd.GetOperands()) { + for (const auto op : listOpnd.GetOperands()) { if (op->GetRegisterNumber() == regOpnd.GetRegisterNumber()) { return kResUseFirst; } @@ -618,8 +622,10 @@ void PeepOptimizer::Run() { auto *patterMatcher = peepOptMemPool->New(cgFunc, peepOptMemPool); patterMatcher->InitOpts(); - FOR_ALL_BB(bb, &cgFunc) { - FOR_BB_INSNS_SAFE(insn, bb, nextInsn) { + FOR_ALL_BB(bb, &cgFunc) + { + FOR_BB_INSNS_SAFE(insn, bb, nextInsn) + { if (!insn->IsMachineInstruction()) { continue; } @@ -744,10 +750,20 @@ bool CgPrePeepHole1::PhaseRun(maplebe::CGFunc &f) peep->PrePeepholeOpt1(); return false; } + MAPLE_TRANSFORM_PHASE_REGISTER_CANSKIP(CgPrePeepHole1, prepeephole1) bool CgPeepHole0::PhaseRun(maplebe::CGFunc &f) { + ReachingDefinition *reachingDef = nullptr; + if (Globals::GetInstance()->GetOptimLevel() >= CGOptions::kLevel2) { + reachingDef = GET_ANALYSIS(CgReachingDefinition, f); + } + if (reachingDef == nullptr || !f.GetRDStatus()) { + GetAnalysisInfoHook()->ForceEraseAnalysisPhase(f.GetUniqueID(), &CgReachingDefinition::id); + return false; + } + auto *peep = GetPhaseMemPool()->New(&f); CHECK_FATAL(peep != nullptr, "PeepHoleOptimizer instance create failure"); peep->Peephole0(); diff --git a/ecmascript/compiler/codegen/maple/maple_be/src/cg/x86_64/x64_cfgo.cpp b/ecmascript/compiler/codegen/maple/maple_be/src/cg/x86_64/x64_cfgo.cpp index 80e8326eb801b4e1835efc699c5662779e0531bc..b6e64a82865d1a5be9ce683f492a252ea9c331dd 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/src/cg/x86_64/x64_cfgo.cpp +++ b/ecmascript/compiler/codegen/maple/maple_be/src/cg/x86_64/x64_cfgo.cpp @@ -20,11 +20,7 @@ namespace maplebe { /* Initialize cfg optimization patterns */ void X64CFGOptimizer::InitOptimizePatterns() { - /* disable the pass that conflicts with cfi */ - if (!cgFunc->GenCfi()) { - diffPassPatterns.emplace_back(memPool->New(*cgFunc)); - } - diffPassPatterns.emplace_back(memPool->New(*cgFunc)); + // The SequentialJumpPattern is not supported until the interface is optimized. diffPassPatterns.emplace_back(memPool->New(*cgFunc)); diffPassPatterns.emplace_back(memPool->New(*cgFunc)); diffPassPatterns.emplace_back(memPool->New(*cgFunc)); diff --git a/ecmascript/compiler/codegen/maple/maple_be/src/cg/x86_64/x64_optimize_common.cpp b/ecmascript/compiler/codegen/maple/maple_be/src/cg/x86_64/x64_optimize_common.cpp index 1049916d3b90857a9f68cecb665fbd1edaefd7f9..4a0094483be4b50a96d2136a6a12ddd1cd39d80e 100644 --- a/ecmascript/compiler/codegen/maple/maple_be/src/cg/x86_64/x64_optimize_common.cpp +++ b/ecmascript/compiler/codegen/maple/maple_be/src/cg/x86_64/x64_optimize_common.cpp @@ -16,6 +16,7 @@ #include "x64_optimize_common.h" #include "x64_cgfunc.h" #include "cgbb.h" +#include "cg.h" namespace maplebe { void X64InsnVisitor::ModifyJumpTarget(Operand &targetOperand, BB &bb) @@ -80,81 +81,95 @@ LabelIdx X64InsnVisitor::GetJumpLabel(const Insn &insn) const bool X64InsnVisitor::IsCompareInsn(const Insn &insn) const { switch (insn.GetMachineOpcode()) { - case MOP_cmpb_r_r: - case MOP_cmpb_m_r: - case MOP_cmpb_i_r: - case MOP_cmpb_r_m: - case MOP_cmpb_i_m: - case MOP_cmpw_r_r: - case MOP_cmpw_m_r: - case MOP_cmpw_i_r: - case MOP_cmpw_r_m: - case MOP_cmpw_i_m: - case MOP_cmpl_r_r: - case MOP_cmpl_m_r: - case MOP_cmpl_i_r: - case MOP_cmpl_r_m: - case MOP_cmpl_i_m: - case MOP_cmpq_r_r: - case MOP_cmpq_m_r: - case MOP_cmpq_i_r: - case MOP_cmpq_r_m: - case MOP_cmpq_i_m: - case MOP_testq_r_r: + case x64::MOP_cmpb_r_r: + case x64::MOP_cmpb_m_r: + case x64::MOP_cmpb_i_r: + case x64::MOP_cmpb_r_m: + case x64::MOP_cmpb_i_m: + case x64::MOP_cmpw_r_r: + case x64::MOP_cmpw_m_r: + case x64::MOP_cmpw_i_r: + case x64::MOP_cmpw_r_m: + case x64::MOP_cmpw_i_m: + case x64::MOP_cmpl_r_r: + case x64::MOP_cmpl_m_r: + case x64::MOP_cmpl_i_r: + case x64::MOP_cmpl_r_m: + case x64::MOP_cmpl_i_m: + case x64::MOP_cmpq_r_r: + case x64::MOP_cmpq_m_r: + case x64::MOP_cmpq_i_r: + case x64::MOP_cmpq_r_m: + case x64::MOP_cmpq_i_m: + case x64::MOP_testq_r_r: return true; default: return false; } } +bool X64InsnVisitor::IsSimpleJumpInsn(const Insn &insn) const +{ + return (insn.GetMachineOpcode() == x64::MOP_jmpq_l); +} + bool X64InsnVisitor::IsCompareAndBranchInsn(const Insn &insn) const { return false; } +bool X64InsnVisitor::IsTestAndSetCCInsn(const Insn &insn) const +{ + return false; +} + +bool X64InsnVisitor::IsTestAndBranchInsn(const Insn &insn) const +{ + return false; +} + bool X64InsnVisitor::IsAddOrSubInsn(const Insn &insn) const { switch (insn.GetMachineOpcode()) { - case MOP_addb_r_r: - case MOP_addw_r_r: - case MOP_addl_r_r: - case MOP_addq_r_r: - case MOP_addb_m_r: - case MOP_addw_m_r: - case MOP_addl_m_r: - case MOP_addq_m_r: - case MOP_addb_i_r: - case MOP_addw_i_r: - case MOP_addl_i_r: - case MOP_addq_i_r: - case MOP_addb_r_m: - case MOP_addw_r_m: - case MOP_addl_r_m: - case MOP_addq_r_m: - case MOP_addb_i_m: - case MOP_addw_i_m: - case MOP_addl_i_m: - case MOP_addq_i_m: - case MOP_subb_r_r: - case MOP_subw_r_r: - case MOP_subl_r_r: - case MOP_subq_r_r: - case MOP_subb_m_r: - case MOP_subw_m_r: - case MOP_subl_m_r: - case MOP_subq_m_r: - case MOP_subb_i_r: - case MOP_subw_i_r: - case MOP_subl_i_r: - case MOP_subq_i_r: - case MOP_subb_r_m: - case MOP_subw_r_m: - case MOP_subl_r_m: - case MOP_subq_r_m: - case MOP_subb_i_m: - case MOP_subw_i_m: - case MOP_subl_i_m: - case MOP_subq_i_m: + case x64::MOP_addb_r_r: + case x64::MOP_addw_r_r: + case x64::MOP_addl_r_r: + case x64::MOP_addq_r_r: + case x64::MOP_addb_m_r: + case x64::MOP_addw_m_r: + case x64::MOP_addl_m_r: + case x64::MOP_addq_m_r: + case x64::MOP_addb_i_r: + case x64::MOP_addw_i_r: + case x64::MOP_addl_i_r: + case x64::MOP_addq_i_r: + case x64::MOP_addb_r_m: + case x64::MOP_addw_r_m: + case x64::MOP_addl_r_m: + case x64::MOP_addq_r_m: + case x64::MOP_addb_i_m: + case x64::MOP_addw_i_m: + case x64::MOP_addl_i_m: + case x64::MOP_addq_i_m: + case x64::MOP_subb_r_r: + case x64::MOP_subw_r_r: + case x64::MOP_subl_r_r: + case x64::MOP_subq_r_r: + case x64::MOP_subb_m_r: + case x64::MOP_subw_m_r: + case x64::MOP_subl_m_r: + case x64::MOP_subq_m_r: + case x64::MOP_subb_i_r: + case x64::MOP_subw_i_r: + case x64::MOP_subl_i_r: + case x64::MOP_subq_i_r: + case x64::MOP_subb_r_m: + case x64::MOP_subw_r_m: + case x64::MOP_subl_r_m: + case x64::MOP_subq_r_m: + case x64::MOP_subb_i_m: + case x64::MOP_subw_i_m: + case x64::MOP_subl_i_m: return true; default: return false; @@ -165,4 +180,36 @@ RegOperand *X64InsnVisitor::CreateVregFromReg(const RegOperand &pReg) { return &GetCGFunc()->GetOpndBuilder()->CreateVReg(pReg.GetRegisterNumber(), pReg.GetSize(), pReg.GetRegisterType()); } + +void X64InsnVisitor::ReTargetSuccBB(BB &bb, LabelIdx newTarget) const +{ + DEBUG_ASSERT(false, "not implement in X86_64"); + (void)bb; + (void)newTarget; + return; +} + +void X64InsnVisitor::FlipIfBB(BB &bb, LabelIdx ftLabel) const +{ + DEBUG_ASSERT(false, "not implement in X86_64"); + (void)bb; + (void)ftLabel; + return; +} + +BB *X64InsnVisitor::CreateGotoBBAfterCondBB(BB &bb, BB &fallthru, bool isTargetFallthru) const +{ + DEBUG_ASSERT(false, "not implement in X86_64"); + (void)bb; + (void)fallthru; + (void)isTargetFallthru; + return nullptr; +} + +void X64InsnVisitor::ModifyFathruBBToGotoBB(BB &bb, LabelIdx labelIdx) const +{ + DEBUG_ASSERT(false, "not implement in X86_64"); + (void)bb; + return; +} } /* namespace maplebe */ diff --git a/ecmascript/compiler/codegen/maple/maple_util/include/mem_reference_table.h b/ecmascript/compiler/codegen/maple/maple_util/include/mem_reference_table.h new file mode 100644 index 0000000000000000000000000000000000000000..99c5d10b7d78cc3e8380d2a953356a93787cde4e --- /dev/null +++ b/ecmascript/compiler/codegen/maple/maple_util/include/mem_reference_table.h @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MAPLE_IR_INCLUDE_MEM_REFERENCE_TABLE_H +#define MAPLE_IR_INCLUDE_MEM_REFERENCE_TABLE_H + +#include "mempool_allocator.h" +#include "mir_module.h" +#include "orig_symbol.h" + +namespace maple { + +using MemDefUseSet = MapleUnorderedSet; + +class MemDefUse { +public: + explicit MemDefUse(MapleAllocator &allocator) : defSet(allocator.Adapter()), useSet(allocator.Adapter()) {} + ~MemDefUse(); + + const MemDefUseSet &GetDefSet() const + { + return defSet; + } + + MemDefUseSet &GetDefSet() + { + return defSet; + } + + const MemDefUseSet &GetUseSet() const + { + return useSet; + } + + MemDefUseSet &GetUseSet() + { + return useSet; + } + + void SetIndependent() + { + isIndependent = true; + } + + bool IsIndependent() const + { + return isIndependent; + } + + void MergeOthers(MemDefUse &rhs) + { + if (this == &rhs) { + return; + } + defSet.insert(rhs.defSet.begin(), rhs.defSet.end()); + useSet.insert(rhs.useSet.begin(), rhs.useSet.end()); + isIndependent |= rhs.isIndependent; + } + +private: + MemDefUseSet defSet; + MemDefUseSet useSet; + // The field is set TRUE to indicate that the callee-save and callee-reload instructions do not + // conflict with any other memory access instructions. + bool isIndependent = false; +}; + +using MemDefUsePart = MapleUnorderedMap; +using OstTable = MapleVector; +class MemReferenceTable { +public: + MemReferenceTable(MapleAllocator &allocator, MIRFunction &func) + : allocator(allocator), func(func), ostTable(allocator.Adapter()), memDefUsePart(allocator.Adapter()) + { + } + ~MemReferenceTable() {} + + MIRFunction &GetFunction() + { + return func; + } + + MemDefUsePart &GetMemDefUsePart() + { + return memDefUsePart; + } + + OstTable &GetOstTable() + { + return ostTable; + } + + MemDefUse *GetOrCreateMemDefUseFromBaseNode(BaseNode *node) + { + auto iter = memDefUsePart.find(node); + if (iter != memDefUsePart.end()) { + return iter->second; + } + auto *newDefUse = allocator.New(allocator); + memDefUsePart[node] = newDefUse; + return newDefUse; + } + +private: + MapleAllocator &allocator; + MIRFunction &func; + OstTable ostTable; + MemDefUsePart memDefUsePart; +}; +} // namespace maple +#endif // MAPLE_IR_INCLUDE_MEM_REFERENCE_TABLE_H diff --git a/ecmascript/compiler/codegen/maple/maple_util/include/mpl_logging.h b/ecmascript/compiler/codegen/maple/maple_util/include/mpl_logging.h index dcd62ce4c612d3b1f0067bbaf2e8bead364dc47d..ca7d152fa7a7403469e1427f4a3b6f55000b4de1 100644 --- a/ecmascript/compiler/codegen/maple/maple_util/include/mpl_logging.h +++ b/ecmascript/compiler/codegen/maple/maple_util/include/mpl_logging.h @@ -213,6 +213,13 @@ private: DEBUG_STMT(CHECK(cond, fmt, ##__VA_ARGS__)); \ } while (0) +// To shut down the codecheck warning: boolean condition for 'if' always evaluates to 'true' +#define CHECK_FATAL_FALSE(fmt, ...) \ + do { \ + maple::logInfo.EmitErrorMessage("false", __FILE__, __LINE__, fmt "\n", ##__VA_ARGS__); \ + exit(1); \ + } while (0) + #define CHECK_FATAL(cond, fmt, ...) \ do { \ if (!(cond)) { \