| //===- llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h -------*- C++ -*-===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| // |
| /// \file This file implements GIMatchTableExecutor's `executeMatchTable` |
| /// function. This is implemented in a separate file because the function is |
| /// quite large. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #ifndef LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H |
| #define LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H |
| |
| #include "llvm/ADT/SmallVector.h" |
| #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h" |
| #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" |
| #include "llvm/CodeGen/GlobalISel/Utils.h" |
| #include "llvm/CodeGen/MachineInstrBuilder.h" |
| #include "llvm/CodeGen/MachineOperand.h" |
| #include "llvm/CodeGen/MachineRegisterInfo.h" |
| #include "llvm/CodeGen/RegisterBankInfo.h" |
| #include "llvm/CodeGen/TargetInstrInfo.h" |
| #include "llvm/CodeGen/TargetOpcodes.h" |
| #include "llvm/CodeGen/TargetRegisterInfo.h" |
| #include "llvm/IR/Constants.h" |
| #include "llvm/IR/DataLayout.h" |
| #include "llvm/IR/Type.h" |
| #include "llvm/Support/CodeGenCoverage.h" |
| #include "llvm/Support/Debug.h" |
| #include "llvm/Support/ErrorHandling.h" |
| #include "llvm/Support/raw_ostream.h" |
| #include <cassert> |
| #include <cstddef> |
| #include <cstdint> |
| |
| namespace llvm { |
| |
| template <class TgtExecutor, class PredicateBitset, class ComplexMatcherMemFn, |
| class CustomRendererFn> |
| bool GIMatchTableExecutor::executeMatchTable( |
| TgtExecutor &Exec, NewMIVector &OutMIs, MatcherState &State, |
| const ExecInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn> |
| &ExecInfo, |
| const int64_t *MatchTable, const TargetInstrInfo &TII, |
| MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, |
| const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures, |
| CodeGenCoverage *CoverageInfo, GISelChangeObserver *Observer) const { |
| |
| uint64_t CurrentIdx = 0; |
| SmallVector<uint64_t, 4> OnFailResumeAt; |
| |
| // Bypass the flag check on the instruction, and only look at the MCInstrDesc. |
| bool NoFPException = !State.MIs[0]->getDesc().mayRaiseFPException(); |
| |
| const uint16_t Flags = State.MIs[0]->getFlags(); |
| |
| enum RejectAction { RejectAndGiveUp, RejectAndResume }; |
| auto handleReject = [&]() -> RejectAction { |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": Rejected\n"); |
| if (OnFailResumeAt.empty()) |
| return RejectAndGiveUp; |
| CurrentIdx = OnFailResumeAt.pop_back_val(); |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " (" |
| << OnFailResumeAt.size() << " try-blocks remain)\n"); |
| return RejectAndResume; |
| }; |
| |
| auto propagateFlags = [=](NewMIVector &OutMIs) { |
| for (auto MIB : OutMIs) { |
| // Set the NoFPExcept flag when no original matched instruction could |
| // raise an FP exception, but the new instruction potentially might. |
| uint16_t MIBFlags = Flags; |
| if (NoFPException && MIB->mayRaiseFPException()) |
| MIBFlags |= MachineInstr::NoFPExcept; |
| MIB.setMIFlags(MIBFlags); |
| } |
| |
| return true; |
| }; |
| |
| while (true) { |
| assert(CurrentIdx != ~0u && "Invalid MatchTable index"); |
| int64_t MatcherOpcode = MatchTable[CurrentIdx++]; |
| switch (MatcherOpcode) { |
| case GIM_Try: { |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": Begin try-block\n"); |
| OnFailResumeAt.push_back(MatchTable[CurrentIdx++]); |
| break; |
| } |
| |
| case GIM_RecordInsn: |
| case GIM_RecordInsnIgnoreCopies: { |
| int64_t NewInsnID = MatchTable[CurrentIdx++]; |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t OpIdx = MatchTable[CurrentIdx++]; |
| |
| // As an optimisation we require that MIs[0] is always the root. Refuse |
| // any attempt to modify it. |
| assert(NewInsnID != 0 && "Refusing to modify MIs[0]"); |
| |
| MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); |
| if (!MO.isReg()) { |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": Not a register\n"); |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| break; |
| } |
| if (MO.getReg().isPhysical()) { |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": Is a physical register\n"); |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| break; |
| } |
| |
| MachineInstr *NewMI; |
| if (MatcherOpcode == GIM_RecordInsnIgnoreCopies) |
| NewMI = getDefIgnoringCopies(MO.getReg(), MRI); |
| else |
| NewMI = MRI.getVRegDef(MO.getReg()); |
| |
| if ((size_t)NewInsnID < State.MIs.size()) |
| State.MIs[NewInsnID] = NewMI; |
| else { |
| assert((size_t)NewInsnID == State.MIs.size() && |
| "Expected to store MIs in order"); |
| State.MIs.push_back(NewMI); |
| } |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": MIs[" << NewInsnID |
| << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx |
| << ")\n"); |
| break; |
| } |
| |
| case GIM_CheckFeatures: { |
| int64_t ExpectedBitsetID = MatchTable[CurrentIdx++]; |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx |
| << ": GIM_CheckFeatures(ExpectedBitsetID=" |
| << ExpectedBitsetID << ")\n"); |
| if ((AvailableFeatures & ExecInfo.FeatureBitsets[ExpectedBitsetID]) != |
| ExecInfo.FeatureBitsets[ExpectedBitsetID]) { |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| } |
| break; |
| } |
| case GIM_CheckOpcode: |
| case GIM_CheckOpcodeIsEither: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t Expected0 = MatchTable[CurrentIdx++]; |
| int64_t Expected1 = -1; |
| if (MatcherOpcode == GIM_CheckOpcodeIsEither) |
| Expected1 = MatchTable[CurrentIdx++]; |
| |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| unsigned Opcode = State.MIs[InsnID]->getOpcode(); |
| |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID |
| << "], ExpectedOpcode=" << Expected0; |
| if (MatcherOpcode == GIM_CheckOpcodeIsEither) dbgs() |
| << " || " << Expected1; |
| dbgs() << ") // Got=" << Opcode << "\n";); |
| |
| if (Opcode != Expected0 && Opcode != Expected1) { |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| } |
| break; |
| } |
| case GIM_SwitchOpcode: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t LowerBound = MatchTable[CurrentIdx++]; |
| int64_t UpperBound = MatchTable[CurrentIdx++]; |
| int64_t Default = MatchTable[CurrentIdx++]; |
| |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| const int64_t Opcode = State.MIs[InsnID]->getOpcode(); |
| |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), { |
| dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], [" |
| << LowerBound << ", " << UpperBound << "), Default=" << Default |
| << ", JumpTable...) // Got=" << Opcode << "\n"; |
| }); |
| if (Opcode < LowerBound || UpperBound <= Opcode) { |
| CurrentIdx = Default; |
| break; |
| } |
| CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)]; |
| if (!CurrentIdx) { |
| CurrentIdx = Default; |
| break; |
| } |
| OnFailResumeAt.push_back(Default); |
| break; |
| } |
| |
| case GIM_SwitchType: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t OpIdx = MatchTable[CurrentIdx++]; |
| int64_t LowerBound = MatchTable[CurrentIdx++]; |
| int64_t UpperBound = MatchTable[CurrentIdx++]; |
| int64_t Default = MatchTable[CurrentIdx++]; |
| |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); |
| |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), { |
| dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID |
| << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", " |
| << UpperBound << "), Default=" << Default |
| << ", JumpTable...) // Got="; |
| if (!MO.isReg()) |
| dbgs() << "Not a VReg\n"; |
| else |
| dbgs() << MRI.getType(MO.getReg()) << "\n"; |
| }); |
| if (!MO.isReg()) { |
| CurrentIdx = Default; |
| break; |
| } |
| const LLT Ty = MRI.getType(MO.getReg()); |
| const auto TyI = ExecInfo.TypeIDMap.find(Ty); |
| if (TyI == ExecInfo.TypeIDMap.end()) { |
| CurrentIdx = Default; |
| break; |
| } |
| const int64_t TypeID = TyI->second; |
| if (TypeID < LowerBound || UpperBound <= TypeID) { |
| CurrentIdx = Default; |
| break; |
| } |
| CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)]; |
| if (!CurrentIdx) { |
| CurrentIdx = Default; |
| break; |
| } |
| OnFailResumeAt.push_back(Default); |
| break; |
| } |
| |
| case GIM_CheckNumOperands: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t Expected = MatchTable[CurrentIdx++]; |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs[" |
| << InsnID << "], Expected=" << Expected << ")\n"); |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| if (State.MIs[InsnID]->getNumOperands() != Expected) { |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| } |
| break; |
| } |
| case GIM_CheckI64ImmPredicate: |
| case GIM_CheckImmOperandPredicate: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t OpIdx = MatcherOpcode == GIM_CheckImmOperandPredicate |
| ? MatchTable[CurrentIdx++] |
| : 1; |
| int64_t Predicate = MatchTable[CurrentIdx++]; |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIM_CheckImmPredicate(MIs[" |
| << InsnID << "]->getOperand(" << OpIdx |
| << "), Predicate=" << Predicate << ")\n"); |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| assert((State.MIs[InsnID]->getOperand(OpIdx).isImm() || |
| State.MIs[InsnID]->getOperand(OpIdx).isCImm()) && |
| "Expected immediate operand"); |
| assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate"); |
| int64_t Value = 0; |
| if (State.MIs[InsnID]->getOperand(OpIdx).isCImm()) |
| Value = State.MIs[InsnID]->getOperand(OpIdx).getCImm()->getSExtValue(); |
| else if (State.MIs[InsnID]->getOperand(OpIdx).isImm()) |
| Value = State.MIs[InsnID]->getOperand(OpIdx).getImm(); |
| else |
| llvm_unreachable("Expected Imm or CImm operand"); |
| |
| if (!testImmPredicate_I64(Predicate, Value)) |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| break; |
| } |
| case GIM_CheckAPIntImmPredicate: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t Predicate = MatchTable[CurrentIdx++]; |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() |
| << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs[" |
| << InsnID << "], Predicate=" << Predicate << ")\n"); |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT && |
| "Expected G_CONSTANT"); |
| assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate"); |
| if (!State.MIs[InsnID]->getOperand(1).isCImm()) |
| llvm_unreachable("Expected Imm or CImm operand"); |
| |
| const APInt &Value = |
| State.MIs[InsnID]->getOperand(1).getCImm()->getValue(); |
| if (!testImmPredicate_APInt(Predicate, Value)) |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| break; |
| } |
| case GIM_CheckAPFloatImmPredicate: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t Predicate = MatchTable[CurrentIdx++]; |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() |
| << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs[" |
| << InsnID << "], Predicate=" << Predicate << ")\n"); |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && |
| "Expected G_FCONSTANT"); |
| assert(State.MIs[InsnID]->getOperand(1).isFPImm() && |
| "Expected FPImm operand"); |
| assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate"); |
| const APFloat &Value = |
| State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF(); |
| |
| if (!testImmPredicate_APFloat(Predicate, Value)) |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| break; |
| } |
| case GIM_CheckIsBuildVectorAllOnes: |
| case GIM_CheckIsBuildVectorAllZeros: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx |
| << ": GIM_CheckBuildVectorAll{Zeros|Ones}(MIs[" |
| << InsnID << "])\n"); |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| |
| const MachineInstr *MI = State.MIs[InsnID]; |
| assert((MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR || |
| MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC) && |
| "Expected G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC"); |
| |
| if (MatcherOpcode == GIM_CheckIsBuildVectorAllOnes) { |
| if (!isBuildVectorAllOnes(*MI, MRI)) { |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| } |
| } else { |
| if (!isBuildVectorAllZeros(*MI, MRI)) { |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| } |
| } |
| |
| break; |
| } |
| case GIM_CheckSimplePredicate: { |
| // Note: we don't check for invalid here because this is purely a hook to |
| // allow some executors (such as the combiner) to check arbitrary, |
| // contextless predicates, such as whether a rule is enabled or not. |
| int64_t Predicate = MatchTable[CurrentIdx++]; |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx |
| << ": GIM_CheckSimplePredicate(Predicate=" |
| << Predicate << ")\n"); |
| assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate"); |
| if (!testSimplePredicate(Predicate)) { |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| } |
| break; |
| } |
| case GIM_CheckCxxInsnPredicate: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t Predicate = MatchTable[CurrentIdx++]; |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() |
| << CurrentIdx << ": GIM_CheckCxxPredicate(MIs[" |
| << InsnID << "], Predicate=" << Predicate << ")\n"); |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate"); |
| |
| if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID], State)) |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| break; |
| } |
| case GIM_CheckHasNoUse: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIM_CheckHasNoUse(MIs[" |
| << InsnID << "]\n"); |
| |
| const MachineInstr *MI = State.MIs[InsnID]; |
| assert(MI && "Used insn before defined"); |
| assert(MI->getNumDefs() > 0 && "No defs"); |
| const Register Res = MI->getOperand(0).getReg(); |
| |
| if (!MRI.use_nodbg_empty(Res)) { |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| } |
| |
| break; |
| } |
| case GIM_CheckAtomicOrdering: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs[" |
| << InsnID << "], " << (uint64_t)Ordering << ")\n"); |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| if (!State.MIs[InsnID]->hasOneMemOperand()) |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| |
| for (const auto &MMO : State.MIs[InsnID]->memoperands()) |
| if (MMO->getMergedOrdering() != Ordering) |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| break; |
| } |
| case GIM_CheckAtomicOrderingOrStrongerThan: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx |
| << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs[" |
| << InsnID << "], " << (uint64_t)Ordering << ")\n"); |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| if (!State.MIs[InsnID]->hasOneMemOperand()) |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| |
| for (const auto &MMO : State.MIs[InsnID]->memoperands()) |
| if (!isAtLeastOrStrongerThan(MMO->getMergedOrdering(), Ordering)) |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| break; |
| } |
| case GIM_CheckAtomicOrderingWeakerThan: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx |
| << ": GIM_CheckAtomicOrderingWeakerThan(MIs[" |
| << InsnID << "], " << (uint64_t)Ordering << ")\n"); |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| if (!State.MIs[InsnID]->hasOneMemOperand()) |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| |
| for (const auto &MMO : State.MIs[InsnID]->memoperands()) |
| if (!isStrongerThan(Ordering, MMO->getMergedOrdering())) |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| break; |
| } |
| case GIM_CheckMemoryAddressSpace: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t MMOIdx = MatchTable[CurrentIdx++]; |
| // This accepts a list of possible address spaces. |
| const int NumAddrSpace = MatchTable[CurrentIdx++]; |
| |
| if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| break; |
| } |
| |
| // Need to still jump to the end of the list of address spaces if we find |
| // a match earlier. |
| const uint64_t LastIdx = CurrentIdx + NumAddrSpace; |
| |
| const MachineMemOperand *MMO = |
| *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); |
| const unsigned MMOAddrSpace = MMO->getAddrSpace(); |
| |
| bool Success = false; |
| for (int I = 0; I != NumAddrSpace; ++I) { |
| unsigned AddrSpace = MatchTable[CurrentIdx++]; |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << "addrspace(" << MMOAddrSpace << ") vs " |
| << AddrSpace << '\n'); |
| |
| if (AddrSpace == MMOAddrSpace) { |
| Success = true; |
| break; |
| } |
| } |
| |
| CurrentIdx = LastIdx; |
| if (!Success && handleReject() == RejectAndGiveUp) |
| return false; |
| break; |
| } |
| case GIM_CheckMemoryAlignment: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t MMOIdx = MatchTable[CurrentIdx++]; |
| unsigned MinAlign = MatchTable[CurrentIdx++]; |
| |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| |
| if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| break; |
| } |
| |
| MachineMemOperand *MMO = |
| *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment" |
| << "(MIs[" << InsnID << "]->memoperands() + " |
| << MMOIdx << ")->getAlignment() >= " << MinAlign |
| << ")\n"); |
| if (MMO->getAlign() < MinAlign && handleReject() == RejectAndGiveUp) |
| return false; |
| |
| break; |
| } |
| case GIM_CheckMemorySizeEqualTo: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t MMOIdx = MatchTable[CurrentIdx++]; |
| uint64_t Size = MatchTable[CurrentIdx++]; |
| |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIM_CheckMemorySizeEqual(MIs[" |
| << InsnID << "]->memoperands() + " << MMOIdx |
| << ", Size=" << Size << ")\n"); |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| |
| if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| break; |
| } |
| |
| MachineMemOperand *MMO = |
| *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); |
| |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << MMO->getSize() |
| << " bytes vs " << Size |
| << " bytes\n"); |
| if (MMO->getSize() != Size) |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| |
| break; |
| } |
| case GIM_CheckMemorySizeEqualToLLT: |
| case GIM_CheckMemorySizeLessThanLLT: |
| case GIM_CheckMemorySizeGreaterThanLLT: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t MMOIdx = MatchTable[CurrentIdx++]; |
| int64_t OpIdx = MatchTable[CurrentIdx++]; |
| |
| DEBUG_WITH_TYPE( |
| TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIM_CheckMemorySize" |
| << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT ? "EqualTo" |
| : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT |
| ? "GreaterThan" |
| : "LessThan") |
| << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx |
| << ", OpIdx=" << OpIdx << ")\n"); |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| |
| MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); |
| if (!MO.isReg()) { |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": Not a register\n"); |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| break; |
| } |
| |
| if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| break; |
| } |
| |
| MachineMemOperand *MMO = |
| *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); |
| |
| unsigned Size = MRI.getType(MO.getReg()).getSizeInBits(); |
| if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT && |
| MMO->getSizeInBits() != Size) { |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT && |
| MMO->getSizeInBits() >= Size) { |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT && |
| MMO->getSizeInBits() <= Size) |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| |
| break; |
| } |
| case GIM_CheckType: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t OpIdx = MatchTable[CurrentIdx++]; |
| int64_t TypeID = MatchTable[CurrentIdx++]; |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID |
| << "]->getOperand(" << OpIdx |
| << "), TypeID=" << TypeID << ")\n"); |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); |
| if (!MO.isReg() || |
| MRI.getType(MO.getReg()) != ExecInfo.TypeObjects[TypeID]) { |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| } |
| break; |
| } |
| case GIM_CheckPointerToAny: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t OpIdx = MatchTable[CurrentIdx++]; |
| uint64_t SizeInBits = MatchTable[CurrentIdx++]; |
| |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs[" |
| << InsnID << "]->getOperand(" << OpIdx |
| << "), SizeInBits=" << SizeInBits << ")\n"); |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); |
| const LLT Ty = MRI.getType(MO.getReg()); |
| |
| // iPTR must be looked up in the target. |
| if (SizeInBits == 0) { |
| MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent(); |
| const unsigned AddrSpace = Ty.getAddressSpace(); |
| SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace); |
| } |
| |
| assert(SizeInBits != 0 && "Pointer size must be known"); |
| |
| if (MO.isReg()) { |
| if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits) |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| } else if (handleReject() == RejectAndGiveUp) |
| return false; |
| |
| break; |
| } |
| case GIM_RecordNamedOperand: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t OpIdx = MatchTable[CurrentIdx++]; |
| uint64_t StoreIdx = MatchTable[CurrentIdx++]; |
| |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIM_RecordNamedOperand(MIs[" |
| << InsnID << "]->getOperand(" << OpIdx |
| << "), StoreIdx=" << StoreIdx << ")\n"); |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| assert(StoreIdx < State.RecordedOperands.size() && "Index out of range"); |
| State.RecordedOperands[StoreIdx] = &State.MIs[InsnID]->getOperand(OpIdx); |
| break; |
| } |
| case GIM_CheckRegBankForClass: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t OpIdx = MatchTable[CurrentIdx++]; |
| int64_t RCEnum = MatchTable[CurrentIdx++]; |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs[" |
| << InsnID << "]->getOperand(" << OpIdx |
| << "), RCEnum=" << RCEnum << ")\n"); |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); |
| if (!MO.isReg() || |
| &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum), |
| MRI.getType(MO.getReg())) != |
| RBI.getRegBank(MO.getReg(), MRI, TRI)) { |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| } |
| break; |
| } |
| |
| case GIM_CheckComplexPattern: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t OpIdx = MatchTable[CurrentIdx++]; |
| int64_t RendererID = MatchTable[CurrentIdx++]; |
| int64_t ComplexPredicateID = MatchTable[CurrentIdx++]; |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": State.Renderers[" << RendererID |
| << "] = GIM_CheckComplexPattern(MIs[" << InsnID |
| << "]->getOperand(" << OpIdx |
| << "), ComplexPredicateID=" << ComplexPredicateID |
| << ")\n"); |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| // FIXME: Use std::invoke() when it's available. |
| ComplexRendererFns Renderer = |
| (Exec.*ExecInfo.ComplexPredicates[ComplexPredicateID])( |
| State.MIs[InsnID]->getOperand(OpIdx)); |
| if (Renderer) |
| State.Renderers[RendererID] = *Renderer; |
| else if (handleReject() == RejectAndGiveUp) |
| return false; |
| break; |
| } |
| |
| case GIM_CheckConstantInt: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t OpIdx = MatchTable[CurrentIdx++]; |
| int64_t Value = MatchTable[CurrentIdx++]; |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs[" |
| << InsnID << "]->getOperand(" << OpIdx |
| << "), Value=" << Value << ")\n"); |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); |
| if (MO.isReg()) { |
| // isOperandImmEqual() will sign-extend to 64-bits, so should we. |
| LLT Ty = MRI.getType(MO.getReg()); |
| // If the type is > 64 bits, it can't be a constant int, so we bail |
| // early because SignExtend64 will assert otherwise. |
| if (Ty.getScalarSizeInBits() > 64) { |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| break; |
| } |
| |
| Value = SignExtend64(Value, Ty.getScalarSizeInBits()); |
| if (!isOperandImmEqual(MO, Value, MRI, /*Splat=*/true)) { |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| } |
| } else if (handleReject() == RejectAndGiveUp) |
| return false; |
| |
| break; |
| } |
| |
| case GIM_CheckLiteralInt: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t OpIdx = MatchTable[CurrentIdx++]; |
| int64_t Value = MatchTable[CurrentIdx++]; |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs[" |
| << InsnID << "]->getOperand(" << OpIdx |
| << "), Value=" << Value << ")\n"); |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); |
| if (MO.isImm() && MO.getImm() == Value) |
| break; |
| |
| if (MO.isCImm() && MO.getCImm()->equalsInt(Value)) |
| break; |
| |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| |
| break; |
| } |
| |
| case GIM_CheckIntrinsicID: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t OpIdx = MatchTable[CurrentIdx++]; |
| int64_t Value = MatchTable[CurrentIdx++]; |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs[" |
| << InsnID << "]->getOperand(" << OpIdx |
| << "), Value=" << Value << ")\n"); |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); |
| if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value) |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| break; |
| } |
| case GIM_CheckCmpPredicate: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t OpIdx = MatchTable[CurrentIdx++]; |
| int64_t Value = MatchTable[CurrentIdx++]; |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs[" |
| << InsnID << "]->getOperand(" << OpIdx |
| << "), Value=" << Value << ")\n"); |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); |
| if (!MO.isPredicate() || MO.getPredicate() != Value) |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| break; |
| } |
| case GIM_CheckIsMBB: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t OpIdx = MatchTable[CurrentIdx++]; |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID |
| << "]->getOperand(" << OpIdx << "))\n"); |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) { |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| } |
| break; |
| } |
| case GIM_CheckIsImm: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t OpIdx = MatchTable[CurrentIdx++]; |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID |
| << "]->getOperand(" << OpIdx << "))\n"); |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| if (!State.MIs[InsnID]->getOperand(OpIdx).isImm()) { |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| } |
| break; |
| } |
| case GIM_CheckIsSafeToFold: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs[" |
| << InsnID << "])\n"); |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) { |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| } |
| break; |
| } |
| case GIM_CheckIsSameOperand: |
| case GIM_CheckIsSameOperandIgnoreCopies: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t OpIdx = MatchTable[CurrentIdx++]; |
| int64_t OtherInsnID = MatchTable[CurrentIdx++]; |
| int64_t OtherOpIdx = MatchTable[CurrentIdx++]; |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs[" |
| << InsnID << "][" << OpIdx << "], MIs[" |
| << OtherInsnID << "][" << OtherOpIdx << "])\n"); |
| assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
| assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined"); |
| |
| MachineOperand &Op = State.MIs[InsnID]->getOperand(OpIdx); |
| MachineOperand &OtherOp = State.MIs[OtherInsnID]->getOperand(OtherOpIdx); |
| |
| if (MatcherOpcode == GIM_CheckIsSameOperandIgnoreCopies) { |
| if (Op.isReg() && OtherOp.isReg()) { |
| if (getSrcRegIgnoringCopies(Op.getReg(), MRI) == |
| getSrcRegIgnoringCopies(OtherOp.getReg(), MRI)) |
| break; |
| } |
| } |
| |
| if (!Op.isIdenticalTo(OtherOp)) { |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| } |
| break; |
| } |
| case GIM_Reject: |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIM_Reject\n"); |
| if (handleReject() == RejectAndGiveUp) |
| return false; |
| break; |
| case GIR_MutateOpcode: { |
| int64_t OldInsnID = MatchTable[CurrentIdx++]; |
| uint64_t NewInsnID = MatchTable[CurrentIdx++]; |
| int64_t NewOpcode = MatchTable[CurrentIdx++]; |
| if (NewInsnID >= OutMIs.size()) |
| OutMIs.resize(NewInsnID + 1); |
| |
| OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(), |
| State.MIs[OldInsnID]); |
| OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode)); |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs[" |
| << NewInsnID << "], MIs[" << OldInsnID << "], " |
| << NewOpcode << ")\n"); |
| break; |
| } |
| |
| case GIR_BuildMI: { |
| uint64_t NewInsnID = MatchTable[CurrentIdx++]; |
| int64_t Opcode = MatchTable[CurrentIdx++]; |
| if (NewInsnID >= OutMIs.size()) |
| OutMIs.resize(NewInsnID + 1); |
| |
| OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0], |
| MIMetadata(*State.MIs[0]), TII.get(Opcode)); |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs[" |
| << NewInsnID << "], " << Opcode << ")\n"); |
| break; |
| } |
| |
| case GIR_Copy: { |
| int64_t NewInsnID = MatchTable[CurrentIdx++]; |
| int64_t OldInsnID = MatchTable[CurrentIdx++]; |
| int64_t OpIdx = MatchTable[CurrentIdx++]; |
| assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); |
| OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx)); |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() |
| << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID |
| << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n"); |
| break; |
| } |
| |
| case GIR_CopyOrAddZeroReg: { |
| int64_t NewInsnID = MatchTable[CurrentIdx++]; |
| int64_t OldInsnID = MatchTable[CurrentIdx++]; |
| int64_t OpIdx = MatchTable[CurrentIdx++]; |
| int64_t ZeroReg = MatchTable[CurrentIdx++]; |
| assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); |
| MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx); |
| if (isOperandImmEqual(MO, 0, MRI)) |
| OutMIs[NewInsnID].addReg(ZeroReg); |
| else |
| OutMIs[NewInsnID].add(MO); |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs[" |
| << NewInsnID << "], MIs[" << OldInsnID << "], " |
| << OpIdx << ", " << ZeroReg << ")\n"); |
| break; |
| } |
| |
| case GIR_CopySubReg: { |
| int64_t NewInsnID = MatchTable[CurrentIdx++]; |
| int64_t OldInsnID = MatchTable[CurrentIdx++]; |
| int64_t OpIdx = MatchTable[CurrentIdx++]; |
| int64_t SubRegIdx = MatchTable[CurrentIdx++]; |
| assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); |
| OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(), |
| 0, SubRegIdx); |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs[" |
| << NewInsnID << "], MIs[" << OldInsnID << "], " |
| << OpIdx << ", " << SubRegIdx << ")\n"); |
| break; |
| } |
| |
| case GIR_AddImplicitDef: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t RegNum = MatchTable[CurrentIdx++]; |
| auto Flags = (uint64_t)MatchTable[CurrentIdx++]; |
| assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); |
| Flags |= RegState::Implicit; |
| OutMIs[InsnID].addDef(RegNum, Flags); |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs[" |
| << InsnID << "], " << RegNum << ")\n"); |
| break; |
| } |
| |
| case GIR_AddImplicitUse: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t RegNum = MatchTable[CurrentIdx++]; |
| assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); |
| OutMIs[InsnID].addUse(RegNum, RegState::Implicit); |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs[" |
| << InsnID << "], " << RegNum << ")\n"); |
| break; |
| } |
| |
| case GIR_AddRegister: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t RegNum = MatchTable[CurrentIdx++]; |
| uint64_t RegFlags = MatchTable[CurrentIdx++]; |
| assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); |
| OutMIs[InsnID].addReg(RegNum, RegFlags); |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() |
| << CurrentIdx << ": GIR_AddRegister(OutMIs[" << InsnID |
| << "], " << RegNum << ", " << RegFlags << ")\n"); |
| break; |
| } |
| case GIR_SetImplicitDefDead: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t OpIdx = MatchTable[CurrentIdx++]; |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIR_SetImplicitDefDead(OutMIs[" |
| << InsnID << "], OpIdx=" << OpIdx << ")\n"); |
| MachineInstr *MI = OutMIs[InsnID]; |
| assert(MI && "Modifying undefined instruction"); |
| MI->getOperand(MI->getNumExplicitOperands() + OpIdx).setIsDead(); |
| break; |
| } |
| case GIR_AddTempRegister: |
| case GIR_AddTempSubRegister: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t TempRegID = MatchTable[CurrentIdx++]; |
| uint64_t TempRegFlags = MatchTable[CurrentIdx++]; |
| unsigned SubReg = 0; |
| if (MatcherOpcode == GIR_AddTempSubRegister) |
| SubReg = MatchTable[CurrentIdx++]; |
| |
| assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); |
| |
| OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags, |
| SubReg); |
| DEBUG_WITH_TYPE( |
| TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs[" << InsnID |
| << "], TempRegisters[" << TempRegID << "]"; |
| if (SubReg) dbgs() << '.' << TRI.getSubRegIndexName(SubReg); |
| dbgs() << ", " << TempRegFlags << ")\n"); |
| break; |
| } |
| |
| case GIR_AddImm: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t Imm = MatchTable[CurrentIdx++]; |
| assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); |
| OutMIs[InsnID].addImm(Imm); |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID |
| << "], " << Imm << ")\n"); |
| break; |
| } |
| |
| case GIR_AddCImm: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t TypeID = MatchTable[CurrentIdx++]; |
| int64_t Imm = MatchTable[CurrentIdx++]; |
| assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); |
| |
| unsigned Width = ExecInfo.TypeObjects[TypeID].getScalarSizeInBits(); |
| LLVMContext &Ctx = MF->getFunction().getContext(); |
| OutMIs[InsnID].addCImm( |
| ConstantInt::get(IntegerType::get(Ctx, Width), Imm, /*signed*/ true)); |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIR_AddCImm(OutMIs[" << InsnID |
| << "], TypeID=" << TypeID << ", Imm=" << Imm |
| << ")\n"); |
| break; |
| } |
| |
| case GIR_ComplexRenderer: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t RendererID = MatchTable[CurrentIdx++]; |
| assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); |
| for (const auto &RenderOpFn : State.Renderers[RendererID]) |
| RenderOpFn(OutMIs[InsnID]); |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs[" |
| << InsnID << "], " << RendererID << ")\n"); |
| break; |
| } |
| case GIR_ComplexSubOperandRenderer: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t RendererID = MatchTable[CurrentIdx++]; |
| int64_t RenderOpID = MatchTable[CurrentIdx++]; |
| assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); |
| State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]); |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx |
| << ": GIR_ComplexSubOperandRenderer(OutMIs[" |
| << InsnID << "], " << RendererID << ", " |
| << RenderOpID << ")\n"); |
| break; |
| } |
| case GIR_ComplexSubOperandSubRegRenderer: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t RendererID = MatchTable[CurrentIdx++]; |
| int64_t RenderOpID = MatchTable[CurrentIdx++]; |
| int64_t SubRegIdx = MatchTable[CurrentIdx++]; |
| MachineInstrBuilder &MI = OutMIs[InsnID]; |
| assert(MI && "Attempted to add to undefined instruction"); |
| State.Renderers[RendererID][RenderOpID](MI); |
| MI->getOperand(MI->getNumOperands() - 1).setSubReg(SubRegIdx); |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx |
| << ": GIR_ComplexSubOperandSubRegRenderer(OutMIs[" |
| << InsnID << "], " << RendererID << ", " |
| << RenderOpID << ", " << SubRegIdx << ")\n"); |
| break; |
| } |
| |
| case GIR_CopyConstantAsSImm: { |
| int64_t NewInsnID = MatchTable[CurrentIdx++]; |
| int64_t OldInsnID = MatchTable[CurrentIdx++]; |
| assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); |
| assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && |
| "Expected G_CONSTANT"); |
| if (State.MIs[OldInsnID]->getOperand(1).isCImm()) { |
| OutMIs[NewInsnID].addImm( |
| State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue()); |
| } else if (State.MIs[OldInsnID]->getOperand(1).isImm()) |
| OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1)); |
| else |
| llvm_unreachable("Expected Imm or CImm operand"); |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs[" |
| << NewInsnID << "], MIs[" << OldInsnID << "])\n"); |
| break; |
| } |
| |
| // TODO: Needs a test case once we have a pattern that uses this. |
| case GIR_CopyFConstantAsFPImm: { |
| int64_t NewInsnID = MatchTable[CurrentIdx++]; |
| int64_t OldInsnID = MatchTable[CurrentIdx++]; |
| assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); |
| assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && |
| "Expected G_FCONSTANT"); |
| if (State.MIs[OldInsnID]->getOperand(1).isFPImm()) |
| OutMIs[NewInsnID].addFPImm( |
| State.MIs[OldInsnID]->getOperand(1).getFPImm()); |
| else |
| llvm_unreachable("Expected FPImm operand"); |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() |
| << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs[" |
| << NewInsnID << "], MIs[" << OldInsnID << "])\n"); |
| break; |
| } |
| |
| case GIR_CustomRenderer: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t OldInsnID = MatchTable[CurrentIdx++]; |
| int64_t RendererFnID = MatchTable[CurrentIdx++]; |
| assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs[" |
| << InsnID << "], MIs[" << OldInsnID << "], " |
| << RendererFnID << ")\n"); |
| (Exec.*ExecInfo.CustomRenderers[RendererFnID])( |
| OutMIs[InsnID], *State.MIs[OldInsnID], |
| -1); // Not a source operand of the old instruction. |
| break; |
| } |
| case GIR_CustomAction: { |
| int64_t FnID = MatchTable[CurrentIdx++]; |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIR_CustomAction(FnID=" << FnID |
| << ")\n"); |
| assert(FnID > GICXXCustomAction_Invalid && "Expected a valid FnID"); |
| runCustomAction(FnID, State, OutMIs); |
| break; |
| } |
| case GIR_CustomOperandRenderer: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t OldInsnID = MatchTable[CurrentIdx++]; |
| int64_t OpIdx = MatchTable[CurrentIdx++]; |
| int64_t RendererFnID = MatchTable[CurrentIdx++]; |
| assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); |
| |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx |
| << ": GIR_CustomOperandRenderer(OutMIs[" << InsnID |
| << "], MIs[" << OldInsnID << "]->getOperand(" |
| << OpIdx << "), " << RendererFnID << ")\n"); |
| (Exec.*ExecInfo.CustomRenderers[RendererFnID])( |
| OutMIs[InsnID], *State.MIs[OldInsnID], OpIdx); |
| break; |
| } |
| case GIR_ConstrainOperandRC: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| int64_t OpIdx = MatchTable[CurrentIdx++]; |
| int64_t RCEnum = MatchTable[CurrentIdx++]; |
| assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); |
| MachineInstr &I = *OutMIs[InsnID].getInstr(); |
| MachineFunction &MF = *I.getParent()->getParent(); |
| MachineRegisterInfo &MRI = MF.getRegInfo(); |
| const TargetRegisterClass &RC = *TRI.getRegClass(RCEnum); |
| MachineOperand &MO = I.getOperand(OpIdx); |
| constrainOperandRegClass(MF, TRI, MRI, TII, RBI, I, RC, MO); |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs[" |
| << InsnID << "], " << OpIdx << ", " << RCEnum |
| << ")\n"); |
| break; |
| } |
| |
| case GIR_ConstrainSelectedInstOperands: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); |
| constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI, |
| RBI); |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx |
| << ": GIR_ConstrainSelectedInstOperands(OutMIs[" |
| << InsnID << "])\n"); |
| break; |
| } |
| |
| case GIR_MergeMemOperands: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); |
| |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs[" |
| << InsnID << "]"); |
| int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList; |
| while ((MergeInsnID = MatchTable[CurrentIdx++]) != |
| GIU_MergeMemOperands_EndOfList) { |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << ", MIs[" << MergeInsnID << "]"); |
| for (const auto &MMO : State.MIs[MergeInsnID]->memoperands()) |
| OutMIs[InsnID].addMemOperand(MMO); |
| } |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << ")\n"); |
| break; |
| } |
| |
| case GIR_EraseFromParent: { |
| int64_t InsnID = MatchTable[CurrentIdx++]; |
| MachineInstr *MI = State.MIs[InsnID]; |
| assert(MI && "Attempted to erase an undefined instruction"); |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs[" |
| << InsnID << "])\n"); |
| if (Observer) |
| Observer->erasingInstr(*MI); |
| MI->eraseFromParent(); |
| break; |
| } |
| |
| case GIR_MakeTempReg: { |
| int64_t TempRegID = MatchTable[CurrentIdx++]; |
| int64_t TypeID = MatchTable[CurrentIdx++]; |
| |
| State.TempRegisters[TempRegID] = |
| MRI.createGenericVirtualRegister(ExecInfo.TypeObjects[TypeID]); |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": TempRegs[" << TempRegID |
| << "] = GIR_MakeTempReg(" << TypeID << ")\n"); |
| break; |
| } |
| |
| case GIR_Coverage: { |
| int64_t RuleID = MatchTable[CurrentIdx++]; |
| assert(CoverageInfo); |
| CoverageInfo->setCovered(RuleID); |
| |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << CurrentIdx |
| << ": GIR_Coverage(" |
| << RuleID << ")"); |
| break; |
| } |
| |
| case GIR_Done: |
| DEBUG_WITH_TYPE(TgtExecutor::getName(), |
| dbgs() << CurrentIdx << ": GIR_Done\n"); |
| if (Observer) { |
| for (MachineInstr *MI : OutMIs) |
| Observer->createdInstr(*MI); |
| } |
| propagateFlags(OutMIs); |
| return true; |
| default: |
| llvm_unreachable("Unexpected command"); |
| } |
| } |
| } |
| |
| } // end namespace llvm |
| |
| #endif // LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H |
