diff --git a/llvm/docs/GlobalISel/MIRPatterns.rst b/llvm/docs/GlobalISel/MIRPatterns.rst index a3883b14b3e0b..cbbe962dcb818 100644 --- a/llvm/docs/GlobalISel/MIRPatterns.rst +++ b/llvm/docs/GlobalISel/MIRPatterns.rst @@ -183,6 +183,44 @@ Semantics: * The root cannot have any output operands. * The root must be a CodeGenInstruction +Instruction Flags +----------------- + +MIR Patterns support both matching & writing ``MIFlags``. + +.. code-block:: text + :caption: Example + + def Test : GICombineRule< + (defs root:$dst), + (match (G_FOO $dst, $src, (MIFlags FmNoNans, FmNoInfs))), + (apply (G_BAR $dst, $src, (MIFlags FmReassoc)))>; + +In ``apply`` patterns, we also support referring to a matched instruction to +"take" its MIFlags. + +.. code-block:: text + :caption: Example + + ; We match NoNans/NoInfs, but $zext may have more flags. + ; Copy them all into the output instruction, and set Reassoc on the output inst. + def TestCpyFlags : GICombineRule< + (defs root:$dst), + (match (G_FOO $dst, $src, (MIFlags FmNoNans, FmNoInfs)):$zext), + (apply (G_BAR $dst, $src, (MIFlags $zext, FmReassoc)))>; + +The ``not`` operator can be used to check that a flag is NOT present +on a matched instruction, and to remove a flag from a generated instruction. + +.. code-block:: text + :caption: Example + + ; We match NoInfs but we don't want NoNans/Reassoc to be set. $zext may have more flags. + ; Copy them all into the output instruction but remove NoInfs on the output inst. + def TestNot : GICombineRule< + (defs root:$dst), + (match (G_FOO $dst, $src, (MIFlags FmNoInfs, (not FmNoNans, FmReassoc))):$zext), + (apply (G_BAR $dst, $src, (MIFlags $zext, (not FmNoInfs))))>; Limitations ----------- diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h index 6fcd9d09e1863..f5d9f5f40881c 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h @@ -266,6 +266,13 @@ enum { /// - NewOpIdx GIM_CheckCanReplaceReg, + /// Check that a matched instruction has, or doesn't have a MIFlag. + /// + /// - InsnID - Instruction to check. + /// - Flag(s) - (can be one or more flags OR'd together) + GIM_MIFlags, + GIM_MIFlagsNot, + /// Predicates with 'let PredicateCodeUsesOperands = 1' need to examine some /// named operands that will be recorded in RecordedOperands. Names of these /// operands are referenced in predicate argument list. Emitter determines @@ -344,6 +351,20 @@ enum { /// OpIdx starts at 0 for the first implicit def. GIR_SetImplicitDefDead, + /// Set or unset a MIFlag on an instruction. + /// + /// - InsnID - Instruction to modify. + /// - Flag(s) - (can be one or more flags OR'd together) + GIR_SetMIFlags, + GIR_UnsetMIFlags, + + /// Copy the MIFlags of a matched instruction into an + /// output instruction. The flags are OR'd together. + /// + /// - InsnID - Instruction to modify. + /// - OldInsnID - Matched instruction to copy flags from. + GIR_CopyMIFlags, + /// Add a temporary register to the specified instruction /// - InsnID - Instruction ID to modify /// - TempRegID - The temporary register ID to add diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h index 32e2f21d775f3..f0ee76c097bca 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h @@ -88,8 +88,6 @@ bool GIMatchTableExecutor::executeMatchTable( if (Observer) Observer->changedInstr(*MIB); } - - return true; }; // If the index is >= 0, it's an index in the type objects generated by @@ -919,6 +917,32 @@ bool GIMatchTableExecutor::executeMatchTable( } break; } + case GIM_MIFlags: { + int64_t InsnID = MatchTable[CurrentIdx++]; + uint32_t Flags = (uint32_t)MatchTable[CurrentIdx++]; + + DEBUG_WITH_TYPE(TgtExecutor::getName(), + dbgs() << CurrentIdx << ": GIM_MIFlags(MIs[" << InsnID + << "], " << Flags << ")\n"); + if ((State.MIs[InsnID]->getFlags() & Flags) != Flags) { + if (handleReject() == RejectAndGiveUp) + return false; + } + break; + } + case GIM_MIFlagsNot: { + int64_t InsnID = MatchTable[CurrentIdx++]; + uint32_t Flags = (uint32_t)MatchTable[CurrentIdx++]; + + DEBUG_WITH_TYPE(TgtExecutor::getName(), + dbgs() << CurrentIdx << ": GIM_MIFlagsNot(MIs[" << InsnID + << "], " << Flags << ")\n"); + if ((State.MIs[InsnID]->getFlags() & Flags)) { + if (handleReject() == RejectAndGiveUp) + return false; + } + break; + } case GIM_Reject: DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << CurrentIdx << ": GIM_Reject\n"); @@ -1062,6 +1086,39 @@ bool GIMatchTableExecutor::executeMatchTable( MI->getOperand(MI->getNumExplicitOperands() + OpIdx).setIsDead(); break; } + case GIR_SetMIFlags: { + int64_t InsnID = MatchTable[CurrentIdx++]; + uint32_t Flags = (uint32_t)MatchTable[CurrentIdx++]; + + DEBUG_WITH_TYPE(TgtExecutor::getName(), + dbgs() << CurrentIdx << ": GIR_SetMIFlags(OutMIs[" + << InsnID << "], " << Flags << ")\n"); + MachineInstr *MI = OutMIs[InsnID]; + MI->setFlags(MI->getFlags() | Flags); + break; + } + case GIR_UnsetMIFlags: { + int64_t InsnID = MatchTable[CurrentIdx++]; + uint32_t Flags = (uint32_t)MatchTable[CurrentIdx++]; + + DEBUG_WITH_TYPE(TgtExecutor::getName(), + dbgs() << CurrentIdx << ": GIR_UnsetMIFlags(OutMIs[" + << InsnID << "], " << Flags << ")\n"); + MachineInstr *MI = OutMIs[InsnID]; + MI->setFlags(MI->getFlags() & ~Flags); + break; + } + case GIR_CopyMIFlags: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OldInsnID = MatchTable[CurrentIdx++]; + + DEBUG_WITH_TYPE(TgtExecutor::getName(), + dbgs() << CurrentIdx << ": GIR_CopyMIFlags(OutMIs[" + << InsnID << "], MIs[" << OldInsnID << "])\n"); + MachineInstr *MI = OutMIs[InsnID]; + MI->setFlags(MI->getFlags() | State.MIs[OldInsnID]->getFlags()); + break; + } case GIR_AddTempRegister: case GIR_AddTempSubRegister: { int64_t InsnID = MatchTable[CurrentIdx++]; diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td index ee0209eb9e559..76b83cc5df073 100644 --- a/llvm/include/llvm/Target/GlobalISel/Combine.td +++ b/llvm/include/llvm/Target/GlobalISel/Combine.td @@ -164,6 +164,25 @@ def GIReplaceReg : GIBuiltinInst; // TODO: Allow using this directly, like (apply GIEraseRoot) def GIEraseRoot : GIBuiltinInst; +//===----------------------------------------------------------------------===// +// Pattern MIFlags +//===----------------------------------------------------------------------===// + +class MIFlagEnum { + string EnumName = "MachineInstr::" # enumName; +} + +def FmNoNans : MIFlagEnum<"FmNoNans">; +def FmNoInfs : MIFlagEnum<"FmNoInfs">; +def FmNsz : MIFlagEnum<"FmNsz">; +def FmArcp : MIFlagEnum<"FmArcp">; +def FmContract : MIFlagEnum<"FmContract">; +def FmAfn : MIFlagEnum<"FmAfn">; +def FmReassoc : MIFlagEnum<"FmReassoc">; + +def MIFlags; +// def not; -> Already defined as a SDNode + //===----------------------------------------------------------------------===// def extending_load_matchdata : GIDefMatchData<"PreferredTuple">; diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-miflags.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-miflags.td new file mode 100644 index 0000000000000..9f02ff1749365 --- /dev/null +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-miflags.td @@ -0,0 +1,47 @@ +// RUN: llvm-tblgen -I %p/../../../include -gen-global-isel-combiner \ +// RUN: -combiners=MyCombiner %s | \ +// RUN: FileCheck %s + +include "llvm/Target/Target.td" +include "llvm/Target/GlobalISel/Combine.td" + +def MyTargetISA : InstrInfo; +def MyTarget : Target { let InstructionSet = MyTargetISA; } + +def MIFlagsTest : GICombineRule< + (defs root:$dst), + (match (G_SEXT $dst, $tmp), (G_ZEXT $tmp, $src, (MIFlags FmReassoc, FmNsz, (not FmNoNans, FmArcp))):$mi), + (apply (G_MUL $dst, $src, $src, (MIFlags $mi, FmReassoc, (not FmNsz, FmArcp))))>; + +def MyCombiner: GICombiner<"GenMyCombiner", [MIFlagsTest]>; + +// CHECK: const int64_t *GenMyCombiner::getMatchTable() const { +// CHECK-NEXT: constexpr static int64_t MatchTable0[] = { +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 0*/ 49, // Rule ID 0 // +// CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled, +// CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SEXT, +// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: // No operand predicates +// CHECK-NEXT: // MIs[0] tmp +// CHECK-NEXT: GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1] +// CHECK-NEXT: GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_ZEXT, +// CHECK-NEXT: GIM_MIFlags, /*MI*/1, MachineInstr::FmNsz | MachineInstr::FmReassoc, +// CHECK-NEXT: GIM_MIFlagsNot, /*MI*/1, MachineInstr::FmArcp | MachineInstr::FmNoNans, +// CHECK-NEXT: // MIs[1] src +// CHECK-NEXT: // No operand predicates +// CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1, +// CHECK-NEXT: // Combiner Rule #0: MIFlagsTest +// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::G_MUL, +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // src +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // src +// CHECK-NEXT: GIR_CopyMIFlags, /*InsnID*/0, /*OldInsnID*/1, +// CHECK-NEXT: GIR_SetMIFlags, /*InsnID*/0, MachineInstr::FmReassoc, +// CHECK-NEXT: GIR_UnsetMIFlags, /*InsnID*/0, MachineInstr::FmNsz | MachineInstr::FmArcp, +// CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, +// CHECK-NEXT: GIR_Done, +// CHECK-NEXT: // Label 0: @49 +// CHECK-NEXT: GIM_Reject, +// CHECK-NEXT: }; +// CHECK-NEXT: return MatchTable0; +// CHECK-NEXT: } diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/patfrag-errors.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/patfrag-errors.td index 68bec4fa722d1..6f5c2b93609f4 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/patfrag-errors.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/patfrag-errors.td @@ -271,6 +271,19 @@ def root_def_has_multi_defs : GICombineRule< (match (RootDefHasMultiDefs $root, (i32 10))), (apply (COPY $root, (i32 0)))>; +// CHECK: :[[@LINE+2]]:{{[0-9]+}}: error: matching/writing MIFlags is only allowed on CodeGenInstruction patterns +// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: Failed to parse pattern: '(DummyCXXPF ?:$x, (MIFlags FmArcp))' +def miflags_in_pf : GICombineRule< + (defs root:$x), + (match (COPY $x, $y), (DummyCXXPF $x, (MIFlags FmArcp))), + (apply (COPY $x, $y))>; + +// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: '$pf' does not refer to a CodeGenInstruction in MIFlags of '__badtype_for_flagref_in_apply_apply_0' +def badtype_for_flagref_in_apply : GICombineRule< + (defs root:$dst), + (match (G_ZEXT $dst, $src), (DummyCXXPF $src):$pf), + (apply (G_MUL $dst, $src, $src, (MIFlags $pf)))>; + // CHECK: error: Failed to parse one or more rules def MyCombiner: GICombiner<"GenMyCombiner", [ @@ -293,5 +306,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ patfrag_in_apply, patfrag_cannot_be_root, inconsistent_arg_type, - root_def_has_multi_defs + root_def_has_multi_defs, + miflags_in_pf, + badtype_for_flagref_in_apply ]>; diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-errors.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-errors.td index 4f705589b92e9..318438b977dc9 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-errors.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-errors.td @@ -217,6 +217,50 @@ def def_named_imm_apply : GICombineRule< (apply (COPY i32:$tmp, $y), (COPY $x, (i32 0):$tmp):$foo)>; +// CHECK: :[[@LINE+2]]:{{[0-9]+}}: error: MIFlags can only be present once on an instruction +// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: Failed to parse pattern: '(G_ZEXT ?:$dst, ?:$src, (MIFlags FmArcp), (MIFlags FmArcp))' +def multi_miflags : GICombineRule< + (defs root:$dst), + (match (G_ZEXT $dst, $src, (MIFlags FmArcp), (MIFlags FmArcp)):$mi), + (apply (G_MUL $dst, $src, $src))>; + +def NotAMIFlagEnum; + +// CHECK: :[[@LINE+2]]:{{[0-9]+}}: error: 'NotAMIFlagEnum' is not a subclass of 'MIFlagEnum' +// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: Failed to parse pattern: '(G_ZEXT ?:$dst, ?:$src, (MIFlags NotAMIFlagEnum))' +def not_miflagenum_1 : GICombineRule< + (defs root:$dst), + (match (G_ZEXT $dst, $src, (MIFlags NotAMIFlagEnum)):$mi), + (apply (G_MUL $dst, $src, $src))>; + +// CHECK: :[[@LINE+2]]:{{[0-9]+}}: error: 'NotAMIFlagEnum' is not a subclass of 'MIFlagEnum' +// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: Failed to parse pattern: '(G_ZEXT ?:$dst, ?:$src, (MIFlags (not NotAMIFlagEnum)))' +def not_miflagenum_2 : GICombineRule< + (defs root:$dst), + (match (G_ZEXT $dst, $src, (MIFlags (not NotAMIFlagEnum))):$mi), + + (apply (G_MUL $dst, $src, $src))>; + +// CHECK: :[[@LINE+2]]:{{[0-9]+}}: error: matching/writing MIFlags is only allowed on CodeGenInstruction patterns +// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: Failed to parse pattern: '(GIReplaceReg ?:$x, ?:$y, (MIFlags FmArcp))' +def miflags_in_builtin : GICombineRule< + (defs root:$x), + (match (COPY $x, $y)), + (apply (GIReplaceReg $x, $y, (MIFlags FmArcp)))>; + +// CHECK: :[[@LINE+2]]:{{[0-9]+}}: error: 'match' patterns cannot refer to flags from other instructions +// CHECK: :[[@LINE+1]]:{{[0-9]+}}: note: MIFlags in 'mi' refer to: impostor +def using_flagref_in_match : GICombineRule< + (defs root:$dst), + (match (G_ZEXT $dst, $src, (MIFlags $impostor)):$mi), + (apply (G_MUL $dst, $src, $src))>; + +// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: unknown instruction '$impostor' referenced in MIFlags of '__badflagref_in_apply_apply_0' +def badflagref_in_apply : GICombineRule< + (defs root:$dst), + (match (G_ZEXT $dst, $src):$mi), + (apply (G_MUL $dst, $src, $src, (MIFlags $impostor)))>; + // CHECK: error: Failed to parse one or more rules def MyCombiner: GICombiner<"GenMyCombiner", [ @@ -251,5 +295,11 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ bad_mo_type_not_a_valuetype, untyped_new_reg_in_apply, def_named_imm_match, - def_named_imm_apply + def_named_imm_apply, + multi_miflags, + not_miflagenum_1, + not_miflagenum_2, + miflags_in_builtin, + using_flagref_in_match, + badflagref_in_apply ]>; diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-parsing.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-parsing.td index fd41a7d1d7241..26f3bb88da951 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-parsing.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-parsing.td @@ -320,6 +320,28 @@ def TypeOfTest : GICombineRule< (G_ZEXT $tmp, $src)), (apply (G_MUL $dst, (GITypeOf<"$src"> 0), (GITypeOf<"$dst"> -1)))>; + +// CHECK: (CombineRule name:MIFlagsTest id:11 root:dst +// CHECK-NEXT: (MatchPats +// CHECK-NEXT: mi:(CodeGenInstructionPattern G_ZEXT operands:[$dst, $src] (MIFlags (set MachineInstr::FmReassoc) (unset MachineInstr::FmNoNans, MachineInstr::FmArcp))) +// CHECK-NEXT: ) +// CHECK-NEXT: (ApplyPats +// CHECK-NEXT: __MIFlagsTest_apply_0:(CodeGenInstructionPattern G_MUL operands:[$dst, $src, $src] (MIFlags (set MachineInstr::FmReassoc) (unset MachineInstr::FmNsz, MachineInstr::FmArcp) (copy mi))) +// CHECK-NEXT: ) +// CHECK-NEXT: (OperandTable MatchPats +// CHECK-NEXT: dst -> mi +// CHECK-NEXT: src -> +// CHECK-NEXT: ) +// CHECK-NEXT: (OperandTable ApplyPats +// CHECK-NEXT: dst -> __MIFlagsTest_apply_0 +// CHECK-NEXT: src -> +// CHECK-NEXT: ) +// CHECK-NEXT: ) +def MIFlagsTest : GICombineRule< + (defs root:$dst), + (match (G_ZEXT $dst, $src, (MIFlags FmReassoc, (not FmNoNans, FmArcp))):$mi), + (apply (G_MUL $dst, $src, $src, (MIFlags $mi, FmReassoc, (not FmNsz, FmArcp))))>; + def MyCombiner: GICombiner<"GenMyCombiner", [ WipOpcodeTest0, WipOpcodeTest1, @@ -331,5 +353,6 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ PatFragTest1, VariadicsInTest, VariadicsOutTest, - TypeOfTest + TypeOfTest, + MIFlagsTest ]>; diff --git a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp index b4b3db70c076a..071586240e8fb 100644 --- a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp @@ -38,6 +38,7 @@ #include "llvm/ADT/EquivalenceClasses.h" #include "llvm/ADT/Hashing.h" #include "llvm/ADT/MapVector.h" +#include "llvm/ADT/SetVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/ADT/StringSet.h" #include "llvm/Support/CommandLine.h" @@ -79,6 +80,7 @@ constexpr StringLiteral PatFragClassName = "GICombinePatFrag"; constexpr StringLiteral BuiltinInstClassName = "GIBuiltinInst"; constexpr StringLiteral SpecialTyClassName = "GISpecialType"; constexpr StringLiteral TypeOfClassName = "GITypeOf"; +constexpr StringLiteral MIFlagsEnumClassName = "MIFlagEnum"; std::string getIsEnabledPredicateEnumName(unsigned CombinerRuleID) { return "GICXXPred_Simple_IsRule" + to_string(CombinerRuleID) + "Enabled"; @@ -771,6 +773,8 @@ class InstructionPattern : public Pattern { protected: InstructionPattern(unsigned K, StringRef Name) : Pattern(K, Name) {} + virtual void printExtras(raw_ostream &OS) const {} + SmallVector Operands; }; @@ -828,6 +832,8 @@ void InstructionPattern::print(raw_ostream &OS, bool PrintName) const { Sep = ", "; } OS << "]"; + + printExtras(OS); }); } @@ -919,6 +925,25 @@ class OperandTable { //===- CodeGenInstructionPattern ------------------------------------------===// +/// Helper class to contain data associated with a MIFlags operator. +class MIFlagsInfo { +public: + void addSetFlag(const Record *R) { + SetF.insert(R->getValueAsString("EnumName")); + } + void addUnsetFlag(const Record *R) { + UnsetF.insert(R->getValueAsString("EnumName")); + } + void addCopyFlag(StringRef InstName) { CopyF.insert(insertStrRef(InstName)); } + + const auto &set_flags() const { return SetF; } + const auto &unset_flags() const { return UnsetF; } + const auto ©_flags() const { return CopyF; } + +private: + SetVector SetF, UnsetF, CopyF; +}; + /// Matches an instruction, e.g. `G_ADD $x, $y, $z`. class CodeGenInstructionPattern : public InstructionPattern { public: @@ -938,11 +963,17 @@ class CodeGenInstructionPattern : public InstructionPattern { unsigned getNumInstDefs() const override; unsigned getNumInstOperands() const override; + MIFlagsInfo &getOrCreateMIFlagsInfo(); + const MIFlagsInfo *getMIFlagsInfo() const { return FI.get(); } + const CodeGenInstruction &getInst() const { return I; } StringRef getInstName() const override { return I.TheDef->getName(); } private: + void printExtras(raw_ostream &OS) const override; + const CodeGenInstruction &I; + std::unique_ptr FI; }; bool CodeGenInstructionPattern::hasVariadicDefs() const { @@ -976,6 +1007,26 @@ unsigned CodeGenInstructionPattern::getNumInstOperands() const { : NumCGIOps; } +MIFlagsInfo &CodeGenInstructionPattern::getOrCreateMIFlagsInfo() { + if (!FI) + FI = std::make_unique(); + return *FI; +} + +void CodeGenInstructionPattern::printExtras(raw_ostream &OS) const { + if (!FI) + return; + + OS << " (MIFlags"; + if (!FI->set_flags().empty()) + OS << " (set " << join(FI->set_flags(), ", ") << ")"; + if (!FI->unset_flags().empty()) + OS << " (unset " << join(FI->unset_flags(), ", ") << ")"; + if (!FI->copy_flags().empty()) + OS << " (copy " << join(FI->copy_flags(), ", ") << ")"; + OS << ')'; +} + //===- OperandTypeChecker -------------------------------------------------===// /// This is a trivial type checker for all operands in a set of @@ -2214,6 +2265,8 @@ class CombineRuleBuilder { bool parseInstructionPatternOperand(InstructionPattern &IP, const Init *OpInit, const StringInit *OpName) const; + bool parseInstructionPatternMIFlags(InstructionPattern &IP, + const DagInit *Op) const; std::unique_ptr parsePatFragImpl(const Record *Def) const; bool parsePatFragParamList( ArrayRef DiagLoc, const DagInit &OpsList, @@ -2660,6 +2713,19 @@ bool CombineRuleBuilder::checkSemantics() { continue; } + // MIFlags in match cannot use the following syntax: (MIFlags $mi) + if (const auto *CGP = dyn_cast(Pat)) { + if (auto *FI = CGP->getMIFlagsInfo()) { + if (!FI->copy_flags().empty()) { + PrintError( + "'match' patterns cannot refer to flags from other instructions"); + PrintNote("MIFlags in '" + CGP->getName() + + "' refer to: " + join(FI->copy_flags(), ", ")); + return false; + } + } + } + const auto *AOP = dyn_cast(Pat); if (!AOP) continue; @@ -2684,6 +2750,28 @@ bool CombineRuleBuilder::checkSemantics() { return false; } + // Check that the insts mentioned in copy_flags exist. + if (const auto *CGP = dyn_cast(IP)) { + if (auto *FI = CGP->getMIFlagsInfo()) { + for (auto InstName : FI->copy_flags()) { + auto It = MatchPats.find(InstName); + if (It == MatchPats.end()) { + PrintError("unknown instruction '$" + InstName + + "' referenced in MIFlags of '" + CGP->getName() + "'"); + return false; + } + + if (!isa(It->second.get())) { + PrintError( + "'$" + InstName + + "' does not refer to a CodeGenInstruction in MIFlags of '" + + CGP->getName() + "'"); + return false; + } + } + } + } + const auto *BIP = dyn_cast(IP); if (!BIP) continue; @@ -3022,8 +3110,14 @@ CombineRuleBuilder::parseInstructionPattern(const Init &Arg, } for (unsigned K = 0; K < DagPat->getNumArgs(); ++K) { - if (!parseInstructionPatternOperand(*Pat, DagPat->getArg(K), - DagPat->getArgName(K))) + Init *Arg = DagPat->getArg(K); + if (auto *DagArg = getDagWithSpecificOperator(*Arg, "MIFlags")) { + if (!parseInstructionPatternMIFlags(*Pat, DagArg)) + return nullptr; + continue; + } + + if (!parseInstructionPatternOperand(*Pat, Arg, DagPat->getArgName(K))) return nullptr; } @@ -3131,6 +3225,75 @@ bool CombineRuleBuilder::parseInstructionPatternOperand( return ParseErr(); } +bool CombineRuleBuilder::parseInstructionPatternMIFlags( + InstructionPattern &IP, const DagInit *Op) const { + auto *CGIP = dyn_cast(&IP); + if (!CGIP) { + PrintError("matching/writing MIFlags is only allowed on CodeGenInstruction " + "patterns"); + return false; + } + + const auto CheckFlagEnum = [&](const Record *R) { + if (!R->isSubClassOf(MIFlagsEnumClassName)) { + PrintError("'" + R->getName() + "' is not a subclass of '" + + MIFlagsEnumClassName + "'"); + return false; + } + + return true; + }; + + if (CGIP->getMIFlagsInfo()) { + PrintError("MIFlags can only be present once on an instruction"); + return false; + } + + auto &FI = CGIP->getOrCreateMIFlagsInfo(); + for (unsigned K = 0; K < Op->getNumArgs(); ++K) { + const Init *Arg = Op->getArg(K); + + // Match/set a flag: (MIFlags FmNoNans) + if (const auto *Def = dyn_cast(Arg)) { + const Record *R = Def->getDef(); + if (!CheckFlagEnum(R)) + return false; + + FI.addSetFlag(R); + continue; + } + + // Do not match a flag/unset a flag: (MIFlags (not FmNoNans)) + if (const DagInit *NotDag = getDagWithSpecificOperator(*Arg, "not")) { + for (const Init *NotArg : NotDag->getArgs()) { + const DefInit *DefArg = dyn_cast(NotArg); + if (!DefArg) { + PrintError("cannot parse '" + NotArg->getAsUnquotedString() + + "': expected a '" + MIFlagsEnumClassName + "'"); + return false; + } + + const Record *R = DefArg->getDef(); + if (!CheckFlagEnum(R)) + return false; + + FI.addUnsetFlag(R); + continue; + } + + continue; + } + + // Copy flags from a matched instruction: (MIFlags $mi) + if (isa(Arg)) { + FI.addCopyFlag(Op->getArgName(K)->getAsUnquotedString()); + continue; + } + } + + return true; +} + std::unique_ptr CombineRuleBuilder::parsePatFragImpl(const Record *Def) const { auto StackTrace = PrettyStackTraceParse(*Def); @@ -3261,7 +3424,7 @@ bool CombineRuleBuilder::emitMatchPattern(CodeExpansions &CE, auto StackTrace = PrettyStackTraceEmit(RuleDef, &IP); auto &M = addRuleMatcher(Alts); - InstructionMatcher &IM = M.addInstructionMatcher("root"); + InstructionMatcher &IM = M.addInstructionMatcher(IP.getName()); declareInstExpansion(CE, IM, IP.getName()); DenseSet SeenPats; @@ -3693,8 +3856,23 @@ bool CombineRuleBuilder::emitInstructionApplyPattern( DstMI.addRenderer(TempRegID); } - // TODO: works? - DstMI.chooseInsnToMutate(M); + // Render MIFlags + if (const auto *FI = CGIP.getMIFlagsInfo()) { + for (StringRef InstName : FI->copy_flags()) + DstMI.addCopiedMIFlags(M.getInstructionMatcher(InstName)); + for (StringRef F : FI->set_flags()) + DstMI.addSetMIFlags(F); + for (StringRef F : FI->unset_flags()) + DstMI.addUnsetMIFlags(F); + } + + // Don't allow mutating opcodes for GISel combiners. We want a more precise + // handling of MIFlags so we require them to be explicitly preserved. + // + // TODO: We don't mutate very often, if at all in combiners, but it'd be nice + // to re-enable this. We'd then need to always clear MIFlags when mutating + // opcodes, and never mutate an inst that we copy flags from. + // DstMI.chooseInsnToMutate(M); declareInstExpansion(CE, DstMI, P.getName()); return true; @@ -3813,6 +3991,17 @@ bool CombineRuleBuilder::emitCodeGenInstructionMatchPattern( IM.addPredicate(&P.getInst()); declareInstExpansion(CE, IM, P.getName()); + // Check flags if needed. + if (const auto *FI = P.getMIFlagsInfo()) { + assert(FI->copy_flags().empty()); + + if (const auto &SetF = FI->set_flags(); !SetF.empty()) + IM.addPredicate(SetF.getArrayRef()); + if (const auto &UnsetF = FI->unset_flags(); !UnsetF.empty()) + IM.addPredicate(UnsetF.getArrayRef(), + /*CheckNot=*/true); + } + for (const auto &[Idx, OriginalO] : enumerate(P.operands())) { // Remap the operand. This is used when emitting InstructionPatterns inside // PatFrags, so it can remap them to the arguments passed to the pattern. diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/GlobalISelMatchTable.cpp index 6ec85269e6e20..5a4d32a34e2bc 100644 --- a/llvm/utils/TableGen/GlobalISelMatchTable.cpp +++ b/llvm/utils/TableGen/GlobalISelMatchTable.cpp @@ -1541,6 +1541,24 @@ void GenericInstructionPredicateMatcher::emitPredicateOpcodes( << MatchTable::LineBreak; } +//===- MIFlagsInstructionPredicateMatcher ---------------------------------===// + +bool MIFlagsInstructionPredicateMatcher::isIdentical( + const PredicateMatcher &B) const { + if (!InstructionPredicateMatcher::isIdentical(B)) + return false; + const auto &Other = + static_cast(B); + return Flags == Other.Flags && CheckNot == Other.CheckNot; +} + +void MIFlagsInstructionPredicateMatcher::emitPredicateOpcodes( + MatchTable &Table, RuleMatcher &Rule) const { + Table << MatchTable::Opcode(CheckNot ? "GIM_MIFlagsNot" : "GIM_MIFlags") + << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID) + << MatchTable::NamedValue(join(Flags, " | ")) << MatchTable::LineBreak; +} + //===- InstructionMatcher -------------------------------------------------===// OperandMatcher & @@ -1956,6 +1974,30 @@ void BuildMIAction::chooseInsnToMutate(RuleMatcher &Rule) { void BuildMIAction::emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const { + const auto AddMIFlags = [&]() { + for (const InstructionMatcher *IM : CopiedFlags) { + Table << MatchTable::Opcode("GIR_CopyMIFlags") + << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + << MatchTable::Comment("OldInsnID") + << MatchTable::IntValue(IM->getInsnVarID()) + << MatchTable::LineBreak; + } + + if (!SetFlags.empty()) { + Table << MatchTable::Opcode("GIR_SetMIFlags") + << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + << MatchTable::NamedValue(join(SetFlags, " | ")) + << MatchTable::LineBreak; + } + + if (!UnsetFlags.empty()) { + Table << MatchTable::Opcode("GIR_UnsetMIFlags") + << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + << MatchTable::NamedValue(join(UnsetFlags, " | ")) + << MatchTable::LineBreak; + } + }; + if (Matched) { assert(canMutate(Rule, Matched) && "Arranged to mutate an insn that isn't mutatable"); @@ -1992,6 +2034,8 @@ void BuildMIAction::emitActionOpcodes(MatchTable &Table, << MatchTable::LineBreak; } } + + AddMIFlags(); return; } @@ -2039,6 +2083,8 @@ void BuildMIAction::emitActionOpcodes(MatchTable &Table, << MatchTable::LineBreak; } + AddMIFlags(); + // FIXME: This is a hack but it's sufficient for ISel. We'll need to do // better for combines. Particularly when there are multiple match // roots. diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.h b/llvm/utils/TableGen/GlobalISelMatchTable.h index 364f2a1ec725d..469390d731232 100644 --- a/llvm/utils/TableGen/GlobalISelMatchTable.h +++ b/llvm/utils/TableGen/GlobalISelMatchTable.h @@ -790,6 +790,7 @@ class PredicateMatcher { IPM_VectorSplatImm, IPM_NoUse, IPM_GenericPredicate, + IPM_MIFlags, OPM_SameOperand, OPM_ComplexPattern, OPM_IntrinsicID, @@ -1628,6 +1629,28 @@ class GenericInstructionPredicateMatcher : public InstructionPredicateMatcher { RuleMatcher &Rule) const override; }; +class MIFlagsInstructionPredicateMatcher : public InstructionPredicateMatcher { + SmallVector Flags; + bool CheckNot; // false = GIM_MIFlags, true = GIM_MIFlagsNot + +public: + MIFlagsInstructionPredicateMatcher(unsigned InsnVarID, + ArrayRef FlagsToCheck, + bool CheckNot = false) + : InstructionPredicateMatcher(IPM_MIFlags, InsnVarID), + Flags(FlagsToCheck), CheckNot(CheckNot) { + sort(Flags); + } + + static bool classof(const InstructionPredicateMatcher *P) { + return P->getKind() == IPM_MIFlags; + } + + bool isIdentical(const PredicateMatcher &B) const override; + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override; +}; + /// Generates code to check for the absence of use of the result. // TODO? Generalize this to support checking for one use. class NoUsePredicateMatcher : public InstructionPredicateMatcher { @@ -2233,6 +2256,10 @@ class BuildMIAction : public MatchAction { std::vector> OperandRenderers; SmallPtrSet DeadImplicitDefs; + std::vector CopiedFlags; + std::vector SetFlags; + std::vector UnsetFlags; + /// True if the instruction can be built solely by mutating the opcode. bool canMutate(RuleMatcher &Rule, const InstructionMatcher *Insn) const; @@ -2247,6 +2274,12 @@ class BuildMIAction : public MatchAction { unsigned getInsnID() const { return InsnID; } const CodeGenInstruction *getCGI() const { return I; } + void addSetMIFlags(StringRef Flag) { SetFlags.push_back(Flag); } + void addUnsetMIFlags(StringRef Flag) { UnsetFlags.push_back(Flag); } + void addCopiedMIFlags(const InstructionMatcher &IM) { + CopiedFlags.push_back(&IM); + } + void chooseInsnToMutate(RuleMatcher &Rule); void setDeadImplicitDef(Record *R) { DeadImplicitDefs.insert(R); }