1//===- llvm/CodeGen/GlobalISel/LegalizerInfo.h ------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// Interface for Targets to specify which operations they can successfully
10/// select and how the others should be expanded most efficiently.
11///
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H
15#define LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H
16
17#include "llvm/ADT/SmallBitVector.h"
18#include "llvm/ADT/SmallVector.h"
19#include "llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h"
20#include "llvm/CodeGen/MachineMemOperand.h"
21#include "llvm/CodeGen/TargetOpcodes.h"
22#include "llvm/CodeGenTypes/LowLevelType.h"
23#include "llvm/MC/MCInstrDesc.h"
24#include "llvm/Support/AtomicOrdering.h"
25#include "llvm/Support/CommandLine.h"
26#include <cassert>
27#include <cstdint>
28#include <tuple>
29#include <utility>
30
31namespace llvm {
32
33extern cl::opt<bool> DisableGISelLegalityCheck;
34
35class MachineFunction;
36class raw_ostream;
37class LegalizerHelper;
38class LostDebugLocObserver;
39class MachineInstr;
40class MachineRegisterInfo;
41class MCInstrInfo;
42
43namespace LegalizeActions {
44enum LegalizeAction : std::uint8_t {
45 /// The operation is expected to be selectable directly by the target, and
46 /// no transformation is necessary.
47 Legal,
48
49 /// The operation should be synthesized from multiple instructions acting on
50 /// a narrower scalar base-type. For example a 64-bit add might be
51 /// implemented in terms of 32-bit add-with-carry.
52 NarrowScalar,
53
54 /// The operation should be implemented in terms of a wider scalar
55 /// base-type. For example a <2 x s8> add could be implemented as a <2
56 /// x s32> add (ignoring the high bits).
57 WidenScalar,
58
59 /// The (vector) operation should be implemented by splitting it into
60 /// sub-vectors where the operation is legal. For example a <8 x s64> add
61 /// might be implemented as 4 separate <2 x s64> adds. There can be a leftover
62 /// if there are not enough elements for last sub-vector e.g. <7 x s64> add
63 /// will be implemented as 3 separate <2 x s64> adds and one s64 add. Leftover
64 /// types can be avoided by doing MoreElements first.
65 FewerElements,
66
67 /// The (vector) operation should be implemented by widening the input
68 /// vector and ignoring the lanes added by doing so. For example <2 x i8> is
69 /// rarely legal, but you might perform an <8 x i8> and then only look at
70 /// the first two results.
71 MoreElements,
72
73 /// Perform the operation on a different, but equivalently sized type.
74 Bitcast,
75
76 /// The operation itself must be expressed in terms of simpler actions on
77 /// this target. E.g. a SREM replaced by an SDIV and subtraction.
78 Lower,
79
80 /// The operation should be implemented as a call to some kind of runtime
81 /// support library. For example this usually happens on machines that don't
82 /// support floating-point operations natively.
83 Libcall,
84
85 /// The target wants to do something special with this combination of
86 /// operand and type. A callback will be issued when it is needed.
87 Custom,
88
89 /// This operation is completely unsupported on the target. A programming
90 /// error has occurred.
91 Unsupported,
92
93 /// Sentinel value for when no action was found in the specified table.
94 NotFound,
95
96 /// Fall back onto the old rules.
97 /// TODO: Remove this once we've migrated
98 UseLegacyRules,
99};
100} // end namespace LegalizeActions
101raw_ostream &operator<<(raw_ostream &OS, LegalizeActions::LegalizeAction Action);
102
103using LegalizeActions::LegalizeAction;
104
105/// The LegalityQuery object bundles together all the information that's needed
106/// to decide whether a given operation is legal or not.
107/// For efficiency, it doesn't make a copy of Types so care must be taken not
108/// to free it before using the query.
109struct LegalityQuery {
110 unsigned Opcode;
111 ArrayRef<LLT> Types;
112
113 struct MemDesc {
114 LLT MemoryTy;
115 uint64_t AlignInBits;
116 AtomicOrdering Ordering;
117
118 MemDesc() = default;
119 MemDesc(LLT MemoryTy, uint64_t AlignInBits, AtomicOrdering Ordering)
120 : MemoryTy(MemoryTy), AlignInBits(AlignInBits), Ordering(Ordering) {}
121 MemDesc(const MachineMemOperand &MMO)
122 : MemoryTy(MMO.getMemoryType()),
123 AlignInBits(MMO.getAlign().value() * 8),
124 Ordering(MMO.getSuccessOrdering()) {}
125 };
126
127 /// Operations which require memory can use this to place requirements on the
128 /// memory type for each MMO.
129 ArrayRef<MemDesc> MMODescrs;
130
131 constexpr LegalityQuery(unsigned Opcode, const ArrayRef<LLT> Types,
132 const ArrayRef<MemDesc> MMODescrs)
133 : Opcode(Opcode), Types(Types), MMODescrs(MMODescrs) {}
134 constexpr LegalityQuery(unsigned Opcode, const ArrayRef<LLT> Types)
135 : LegalityQuery(Opcode, Types, {}) {}
136
137 raw_ostream &print(raw_ostream &OS) const;
138};
139
140/// The result of a query. It either indicates a final answer of Legal or
141/// Unsupported or describes an action that must be taken to make an operation
142/// more legal.
143struct LegalizeActionStep {
144 /// The action to take or the final answer.
145 LegalizeAction Action;
146 /// If describing an action, the type index to change. Otherwise zero.
147 unsigned TypeIdx;
148 /// If describing an action, the new type for TypeIdx. Otherwise LLT{}.
149 LLT NewType;
150
151 LegalizeActionStep(LegalizeAction Action, unsigned TypeIdx,
152 const LLT NewType)
153 : Action(Action), TypeIdx(TypeIdx), NewType(NewType) {}
154
155 LegalizeActionStep(LegacyLegalizeActionStep Step)
156 : TypeIdx(Step.TypeIdx), NewType(Step.NewType) {
157 switch (Step.Action) {
158 case LegacyLegalizeActions::Legal:
159 Action = LegalizeActions::Legal;
160 break;
161 case LegacyLegalizeActions::NarrowScalar:
162 Action = LegalizeActions::NarrowScalar;
163 break;
164 case LegacyLegalizeActions::WidenScalar:
165 Action = LegalizeActions::WidenScalar;
166 break;
167 case LegacyLegalizeActions::FewerElements:
168 Action = LegalizeActions::FewerElements;
169 break;
170 case LegacyLegalizeActions::MoreElements:
171 Action = LegalizeActions::MoreElements;
172 break;
173 case LegacyLegalizeActions::Bitcast:
174 Action = LegalizeActions::Bitcast;
175 break;
176 case LegacyLegalizeActions::Lower:
177 Action = LegalizeActions::Lower;
178 break;
179 case LegacyLegalizeActions::Libcall:
180 Action = LegalizeActions::Libcall;
181 break;
182 case LegacyLegalizeActions::Custom:
183 Action = LegalizeActions::Custom;
184 break;
185 case LegacyLegalizeActions::Unsupported:
186 Action = LegalizeActions::Unsupported;
187 break;
188 case LegacyLegalizeActions::NotFound:
189 Action = LegalizeActions::NotFound;
190 break;
191 }
192 }
193
194 bool operator==(const LegalizeActionStep &RHS) const {
195 return std::tie(args: Action, args: TypeIdx, args: NewType) ==
196 std::tie(args: RHS.Action, args: RHS.TypeIdx, args: RHS.NewType);
197 }
198};
199
200using LegalityPredicate = std::function<bool (const LegalityQuery &)>;
201using LegalizeMutation =
202 std::function<std::pair<unsigned, LLT>(const LegalityQuery &)>;
203
204namespace LegalityPredicates {
205struct TypePairAndMemDesc {
206 LLT Type0;
207 LLT Type1;
208 LLT MemTy;
209 uint64_t Align;
210
211 bool operator==(const TypePairAndMemDesc &Other) const {
212 return Type0 == Other.Type0 && Type1 == Other.Type1 &&
213 Align == Other.Align && MemTy == Other.MemTy;
214 }
215
216 /// \returns true if this memory access is legal with for the access described
217 /// by \p Other (The alignment is sufficient for the size and result type).
218 bool isCompatible(const TypePairAndMemDesc &Other) const {
219 return Type0 == Other.Type0 && Type1 == Other.Type1 &&
220 Align >= Other.Align &&
221 // FIXME: This perhaps should be stricter, but the current legality
222 // rules are written only considering the size.
223 MemTy.getSizeInBits() == Other.MemTy.getSizeInBits();
224 }
225};
226
227/// True iff P is false.
228template <typename Predicate> Predicate predNot(Predicate P) {
229 return [=](const LegalityQuery &Query) { return !P(Query); };
230}
231
232/// True iff P0 and P1 are true.
233template<typename Predicate>
234Predicate all(Predicate P0, Predicate P1) {
235 return [=](const LegalityQuery &Query) {
236 return P0(Query) && P1(Query);
237 };
238}
239/// True iff all given predicates are true.
240template<typename Predicate, typename... Args>
241Predicate all(Predicate P0, Predicate P1, Args... args) {
242 return all(all(P0, P1), args...);
243}
244
245/// True iff P0 or P1 are true.
246template<typename Predicate>
247Predicate any(Predicate P0, Predicate P1) {
248 return [=](const LegalityQuery &Query) {
249 return P0(Query) || P1(Query);
250 };
251}
252/// True iff any given predicates are true.
253template<typename Predicate, typename... Args>
254Predicate any(Predicate P0, Predicate P1, Args... args) {
255 return any(any(P0, P1), args...);
256}
257
258/// True iff the given type index is the specified type.
259LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit);
260/// True iff the given type index is one of the specified types.
261LegalityPredicate typeInSet(unsigned TypeIdx,
262 std::initializer_list<LLT> TypesInit);
263
264/// True iff the given type index is not the specified type.
265inline LegalityPredicate typeIsNot(unsigned TypeIdx, LLT Type) {
266 return [=](const LegalityQuery &Query) {
267 return Query.Types[TypeIdx] != Type;
268 };
269}
270
271/// True iff the given types for the given pair of type indexes is one of the
272/// specified type pairs.
273LegalityPredicate
274typePairInSet(unsigned TypeIdx0, unsigned TypeIdx1,
275 std::initializer_list<std::pair<LLT, LLT>> TypesInit);
276/// True iff the given types for the given pair of type indexes is one of the
277/// specified type pairs.
278LegalityPredicate typePairAndMemDescInSet(
279 unsigned TypeIdx0, unsigned TypeIdx1, unsigned MMOIdx,
280 std::initializer_list<TypePairAndMemDesc> TypesAndMemDescInit);
281/// True iff the specified type index is a scalar.
282LegalityPredicate isScalar(unsigned TypeIdx);
283/// True iff the specified type index is a vector.
284LegalityPredicate isVector(unsigned TypeIdx);
285/// True iff the specified type index is a pointer (with any address space).
286LegalityPredicate isPointer(unsigned TypeIdx);
287/// True iff the specified type index is a pointer with the specified address
288/// space.
289LegalityPredicate isPointer(unsigned TypeIdx, unsigned AddrSpace);
290
291/// True if the type index is a vector with element type \p EltTy
292LegalityPredicate elementTypeIs(unsigned TypeIdx, LLT EltTy);
293
294/// True iff the specified type index is a scalar that's narrower than the given
295/// size.
296LegalityPredicate scalarNarrowerThan(unsigned TypeIdx, unsigned Size);
297
298/// True iff the specified type index is a scalar that's wider than the given
299/// size.
300LegalityPredicate scalarWiderThan(unsigned TypeIdx, unsigned Size);
301
302/// True iff the specified type index is a scalar or vector with an element type
303/// that's narrower than the given size.
304LegalityPredicate scalarOrEltNarrowerThan(unsigned TypeIdx, unsigned Size);
305
306/// True iff the specified type index is a scalar or a vector with an element
307/// type that's wider than the given size.
308LegalityPredicate scalarOrEltWiderThan(unsigned TypeIdx, unsigned Size);
309
310/// True iff the specified type index is a scalar whose size is not a multiple
311/// of Size.
312LegalityPredicate sizeNotMultipleOf(unsigned TypeIdx, unsigned Size);
313
314/// True iff the specified type index is a scalar whose size is not a power of
315/// 2.
316LegalityPredicate sizeNotPow2(unsigned TypeIdx);
317
318/// True iff the specified type index is a scalar or vector whose element size
319/// is not a power of 2.
320LegalityPredicate scalarOrEltSizeNotPow2(unsigned TypeIdx);
321
322/// True if the total bitwidth of the specified type index is \p Size bits.
323LegalityPredicate sizeIs(unsigned TypeIdx, unsigned Size);
324
325/// True iff the specified type indices are both the same bit size.
326LegalityPredicate sameSize(unsigned TypeIdx0, unsigned TypeIdx1);
327
328/// True iff the first type index has a larger total bit size than second type
329/// index.
330LegalityPredicate largerThan(unsigned TypeIdx0, unsigned TypeIdx1);
331
332/// True iff the first type index has a smaller total bit size than second type
333/// index.
334LegalityPredicate smallerThan(unsigned TypeIdx0, unsigned TypeIdx1);
335
336/// True iff the specified MMO index has a size (rounded to bytes) that is not a
337/// power of 2.
338LegalityPredicate memSizeInBytesNotPow2(unsigned MMOIdx);
339
340/// True iff the specified MMO index has a size that is not an even byte size,
341/// or that even byte size is not a power of 2.
342LegalityPredicate memSizeNotByteSizePow2(unsigned MMOIdx);
343
344/// True iff the specified type index is a vector whose element count is not a
345/// power of 2.
346LegalityPredicate numElementsNotPow2(unsigned TypeIdx);
347/// True iff the specified MMO index has at an atomic ordering of at Ordering or
348/// stronger.
349LegalityPredicate atomicOrderingAtLeastOrStrongerThan(unsigned MMOIdx,
350 AtomicOrdering Ordering);
351} // end namespace LegalityPredicates
352
353namespace LegalizeMutations {
354/// Select this specific type for the given type index.
355LegalizeMutation changeTo(unsigned TypeIdx, LLT Ty);
356
357/// Keep the same type as the given type index.
358LegalizeMutation changeTo(unsigned TypeIdx, unsigned FromTypeIdx);
359
360/// Keep the same scalar or element type as the given type index.
361LegalizeMutation changeElementTo(unsigned TypeIdx, unsigned FromTypeIdx);
362
363/// Keep the same scalar or element type as the given type.
364LegalizeMutation changeElementTo(unsigned TypeIdx, LLT Ty);
365
366/// Keep the same scalar or element type as \p TypeIdx, but take the number of
367/// elements from \p FromTypeIdx.
368LegalizeMutation changeElementCountTo(unsigned TypeIdx, unsigned FromTypeIdx);
369
370/// Keep the same scalar or element type as \p TypeIdx, but take the number of
371/// elements from \p Ty.
372LegalizeMutation changeElementCountTo(unsigned TypeIdx, LLT Ty);
373
374/// Change the scalar size or element size to have the same scalar size as type
375/// index \p FromIndex. Unlike changeElementTo, this discards pointer types and
376/// only changes the size.
377LegalizeMutation changeElementSizeTo(unsigned TypeIdx, unsigned FromTypeIdx);
378
379/// Widen the scalar type or vector element type for the given type index to the
380/// next power of 2.
381LegalizeMutation widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned Min = 0);
382
383/// Widen the scalar type or vector element type for the given type index to
384/// next multiple of \p Size.
385LegalizeMutation widenScalarOrEltToNextMultipleOf(unsigned TypeIdx,
386 unsigned Size);
387
388/// Add more elements to the type for the given type index to the next power of
389/// 2.
390LegalizeMutation moreElementsToNextPow2(unsigned TypeIdx, unsigned Min = 0);
391/// Break up the vector type for the given type index into the element type.
392LegalizeMutation scalarize(unsigned TypeIdx);
393} // end namespace LegalizeMutations
394
395/// A single rule in a legalizer info ruleset.
396/// The specified action is chosen when the predicate is true. Where appropriate
397/// for the action (e.g. for WidenScalar) the new type is selected using the
398/// given mutator.
399class LegalizeRule {
400 LegalityPredicate Predicate;
401 LegalizeAction Action;
402 LegalizeMutation Mutation;
403
404public:
405 LegalizeRule(LegalityPredicate Predicate, LegalizeAction Action,
406 LegalizeMutation Mutation = nullptr)
407 : Predicate(Predicate), Action(Action), Mutation(Mutation) {}
408
409 /// Test whether the LegalityQuery matches.
410 bool match(const LegalityQuery &Query) const {
411 return Predicate(Query);
412 }
413
414 LegalizeAction getAction() const { return Action; }
415
416 /// Determine the change to make.
417 std::pair<unsigned, LLT> determineMutation(const LegalityQuery &Query) const {
418 if (Mutation)
419 return Mutation(Query);
420 return std::make_pair(x: 0, y: LLT{});
421 }
422};
423
424class LegalizeRuleSet {
425 /// When non-zero, the opcode we are an alias of
426 unsigned AliasOf = 0;
427 /// If true, there is another opcode that aliases this one
428 bool IsAliasedByAnother = false;
429 SmallVector<LegalizeRule, 2> Rules;
430
431#ifndef NDEBUG
432 /// If bit I is set, this rule set contains a rule that may handle (predicate
433 /// or perform an action upon (or both)) the type index I. The uncertainty
434 /// comes from free-form rules executing user-provided lambda functions. We
435 /// conservatively assume such rules do the right thing and cover all type
436 /// indices. The bitset is intentionally 1 bit wider than it absolutely needs
437 /// to be to distinguish such cases from the cases where all type indices are
438 /// individually handled.
439 SmallBitVector TypeIdxsCovered{MCOI::OPERAND_LAST_GENERIC -
440 MCOI::OPERAND_FIRST_GENERIC + 2};
441 SmallBitVector ImmIdxsCovered{MCOI::OPERAND_LAST_GENERIC_IMM -
442 MCOI::OPERAND_FIRST_GENERIC_IMM + 2};
443#endif
444
445 unsigned typeIdx(unsigned TypeIdx) {
446 assert(TypeIdx <=
447 (MCOI::OPERAND_LAST_GENERIC - MCOI::OPERAND_FIRST_GENERIC) &&
448 "Type Index is out of bounds");
449#ifndef NDEBUG
450 TypeIdxsCovered.set(TypeIdx);
451#endif
452 return TypeIdx;
453 }
454
455 void markAllIdxsAsCovered() {
456#ifndef NDEBUG
457 TypeIdxsCovered.set();
458 ImmIdxsCovered.set();
459#endif
460 }
461
462 void add(const LegalizeRule &Rule) {
463 assert(AliasOf == 0 &&
464 "RuleSet is aliased, change the representative opcode instead");
465 Rules.push_back(Elt: Rule);
466 }
467
468 static bool always(const LegalityQuery &) { return true; }
469
470 /// Use the given action when the predicate is true.
471 /// Action should not be an action that requires mutation.
472 LegalizeRuleSet &actionIf(LegalizeAction Action,
473 LegalityPredicate Predicate) {
474 add(Rule: {Predicate, Action});
475 return *this;
476 }
477 /// Use the given action when the predicate is true.
478 /// Action should be an action that requires mutation.
479 LegalizeRuleSet &actionIf(LegalizeAction Action, LegalityPredicate Predicate,
480 LegalizeMutation Mutation) {
481 add(Rule: {Predicate, Action, Mutation});
482 return *this;
483 }
484 /// Use the given action when type index 0 is any type in the given list.
485 /// Action should not be an action that requires mutation.
486 LegalizeRuleSet &actionFor(LegalizeAction Action,
487 std::initializer_list<LLT> Types) {
488 using namespace LegalityPredicates;
489 return actionIf(Action, Predicate: typeInSet(TypeIdx: typeIdx(TypeIdx: 0), TypesInit: Types));
490 }
491 /// Use the given action when type index 0 is any type in the given list.
492 /// Action should be an action that requires mutation.
493 LegalizeRuleSet &actionFor(LegalizeAction Action,
494 std::initializer_list<LLT> Types,
495 LegalizeMutation Mutation) {
496 using namespace LegalityPredicates;
497 return actionIf(Action, Predicate: typeInSet(TypeIdx: typeIdx(TypeIdx: 0), TypesInit: Types), Mutation);
498 }
499 /// Use the given action when type indexes 0 and 1 is any type pair in the
500 /// given list.
501 /// Action should not be an action that requires mutation.
502 LegalizeRuleSet &actionFor(LegalizeAction Action,
503 std::initializer_list<std::pair<LLT, LLT>> Types) {
504 using namespace LegalityPredicates;
505 return actionIf(Action, Predicate: typePairInSet(TypeIdx0: typeIdx(TypeIdx: 0), TypeIdx1: typeIdx(TypeIdx: 1), TypesInit: Types));
506 }
507 /// Use the given action when type indexes 0 and 1 is any type pair in the
508 /// given list.
509 /// Action should be an action that requires mutation.
510 LegalizeRuleSet &actionFor(LegalizeAction Action,
511 std::initializer_list<std::pair<LLT, LLT>> Types,
512 LegalizeMutation Mutation) {
513 using namespace LegalityPredicates;
514 return actionIf(Action, Predicate: typePairInSet(TypeIdx0: typeIdx(TypeIdx: 0), TypeIdx1: typeIdx(TypeIdx: 1), TypesInit: Types),
515 Mutation);
516 }
517 /// Use the given action when type index 0 is any type in the given list and
518 /// imm index 0 is anything. Action should not be an action that requires
519 /// mutation.
520 LegalizeRuleSet &actionForTypeWithAnyImm(LegalizeAction Action,
521 std::initializer_list<LLT> Types) {
522 using namespace LegalityPredicates;
523 immIdx(ImmIdx: 0); // Inform verifier imm idx 0 is handled.
524 return actionIf(Action, Predicate: typeInSet(TypeIdx: typeIdx(TypeIdx: 0), TypesInit: Types));
525 }
526
527 LegalizeRuleSet &actionForTypeWithAnyImm(
528 LegalizeAction Action, std::initializer_list<std::pair<LLT, LLT>> Types) {
529 using namespace LegalityPredicates;
530 immIdx(ImmIdx: 0); // Inform verifier imm idx 0 is handled.
531 return actionIf(Action, Predicate: typePairInSet(TypeIdx0: typeIdx(TypeIdx: 0), TypeIdx1: typeIdx(TypeIdx: 1), TypesInit: Types));
532 }
533
534 /// Use the given action when type indexes 0 and 1 are both in the given list.
535 /// That is, the type pair is in the cartesian product of the list.
536 /// Action should not be an action that requires mutation.
537 LegalizeRuleSet &actionForCartesianProduct(LegalizeAction Action,
538 std::initializer_list<LLT> Types) {
539 using namespace LegalityPredicates;
540 return actionIf(Action, Predicate: all(P0: typeInSet(TypeIdx: typeIdx(TypeIdx: 0), TypesInit: Types),
541 P1: typeInSet(TypeIdx: typeIdx(TypeIdx: 1), TypesInit: Types)));
542 }
543 /// Use the given action when type indexes 0 and 1 are both in their
544 /// respective lists.
545 /// That is, the type pair is in the cartesian product of the lists
546 /// Action should not be an action that requires mutation.
547 LegalizeRuleSet &
548 actionForCartesianProduct(LegalizeAction Action,
549 std::initializer_list<LLT> Types0,
550 std::initializer_list<LLT> Types1) {
551 using namespace LegalityPredicates;
552 return actionIf(Action, Predicate: all(P0: typeInSet(TypeIdx: typeIdx(TypeIdx: 0), TypesInit: Types0),
553 P1: typeInSet(TypeIdx: typeIdx(TypeIdx: 1), TypesInit: Types1)));
554 }
555 /// Use the given action when type indexes 0, 1, and 2 are all in their
556 /// respective lists.
557 /// That is, the type triple is in the cartesian product of the lists
558 /// Action should not be an action that requires mutation.
559 LegalizeRuleSet &actionForCartesianProduct(
560 LegalizeAction Action, std::initializer_list<LLT> Types0,
561 std::initializer_list<LLT> Types1, std::initializer_list<LLT> Types2) {
562 using namespace LegalityPredicates;
563 return actionIf(Action, Predicate: all(P0: typeInSet(TypeIdx: typeIdx(TypeIdx: 0), TypesInit: Types0),
564 P1: all(P0: typeInSet(TypeIdx: typeIdx(TypeIdx: 1), TypesInit: Types1),
565 P1: typeInSet(TypeIdx: typeIdx(TypeIdx: 2), TypesInit: Types2))));
566 }
567
568public:
569 LegalizeRuleSet() = default;
570
571 bool isAliasedByAnother() { return IsAliasedByAnother; }
572 void setIsAliasedByAnother() { IsAliasedByAnother = true; }
573 void aliasTo(unsigned Opcode) {
574 assert((AliasOf == 0 || AliasOf == Opcode) &&
575 "Opcode is already aliased to another opcode");
576 assert(Rules.empty() && "Aliasing will discard rules");
577 AliasOf = Opcode;
578 }
579 unsigned getAlias() const { return AliasOf; }
580
581 unsigned immIdx(unsigned ImmIdx) {
582 assert(ImmIdx <= (MCOI::OPERAND_LAST_GENERIC_IMM -
583 MCOI::OPERAND_FIRST_GENERIC_IMM) &&
584 "Imm Index is out of bounds");
585#ifndef NDEBUG
586 ImmIdxsCovered.set(ImmIdx);
587#endif
588 return ImmIdx;
589 }
590
591 /// The instruction is legal if predicate is true.
592 LegalizeRuleSet &legalIf(LegalityPredicate Predicate) {
593 // We have no choice but conservatively assume that the free-form
594 // user-provided Predicate properly handles all type indices:
595 markAllIdxsAsCovered();
596 return actionIf(Action: LegalizeAction::Legal, Predicate);
597 }
598 /// The instruction is legal when type index 0 is any type in the given list.
599 LegalizeRuleSet &legalFor(std::initializer_list<LLT> Types) {
600 return actionFor(Action: LegalizeAction::Legal, Types);
601 }
602 /// The instruction is legal when type indexes 0 and 1 is any type pair in the
603 /// given list.
604 LegalizeRuleSet &legalFor(std::initializer_list<std::pair<LLT, LLT>> Types) {
605 return actionFor(Action: LegalizeAction::Legal, Types);
606 }
607 /// The instruction is legal when type index 0 is any type in the given list
608 /// and imm index 0 is anything.
609 LegalizeRuleSet &legalForTypeWithAnyImm(std::initializer_list<LLT> Types) {
610 markAllIdxsAsCovered();
611 return actionForTypeWithAnyImm(Action: LegalizeAction::Legal, Types);
612 }
613
614 LegalizeRuleSet &legalForTypeWithAnyImm(
615 std::initializer_list<std::pair<LLT, LLT>> Types) {
616 markAllIdxsAsCovered();
617 return actionForTypeWithAnyImm(Action: LegalizeAction::Legal, Types);
618 }
619
620 /// The instruction is legal when type indexes 0 and 1 along with the memory
621 /// size and minimum alignment is any type and size tuple in the given list.
622 LegalizeRuleSet &legalForTypesWithMemDesc(
623 std::initializer_list<LegalityPredicates::TypePairAndMemDesc>
624 TypesAndMemDesc) {
625 return actionIf(Action: LegalizeAction::Legal,
626 Predicate: LegalityPredicates::typePairAndMemDescInSet(
627 TypeIdx0: typeIdx(TypeIdx: 0), TypeIdx1: typeIdx(TypeIdx: 1), /*MMOIdx*/ MMOIdx: 0, TypesAndMemDescInit: TypesAndMemDesc));
628 }
629 /// The instruction is legal when type indexes 0 and 1 are both in the given
630 /// list. That is, the type pair is in the cartesian product of the list.
631 LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types) {
632 return actionForCartesianProduct(Action: LegalizeAction::Legal, Types);
633 }
634 /// The instruction is legal when type indexes 0 and 1 are both their
635 /// respective lists.
636 LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types0,
637 std::initializer_list<LLT> Types1) {
638 return actionForCartesianProduct(Action: LegalizeAction::Legal, Types0, Types1);
639 }
640 /// The instruction is legal when type indexes 0, 1, and 2 are both their
641 /// respective lists.
642 LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types0,
643 std::initializer_list<LLT> Types1,
644 std::initializer_list<LLT> Types2) {
645 return actionForCartesianProduct(Action: LegalizeAction::Legal, Types0, Types1,
646 Types2);
647 }
648
649 LegalizeRuleSet &alwaysLegal() {
650 using namespace LegalizeMutations;
651 markAllIdxsAsCovered();
652 return actionIf(Action: LegalizeAction::Legal, Predicate: always);
653 }
654
655 /// The specified type index is coerced if predicate is true.
656 LegalizeRuleSet &bitcastIf(LegalityPredicate Predicate,
657 LegalizeMutation Mutation) {
658 // We have no choice but conservatively assume that lowering with a
659 // free-form user provided Predicate properly handles all type indices:
660 markAllIdxsAsCovered();
661 return actionIf(Action: LegalizeAction::Bitcast, Predicate, Mutation);
662 }
663
664 /// The instruction is lowered.
665 LegalizeRuleSet &lower() {
666 using namespace LegalizeMutations;
667 // We have no choice but conservatively assume that predicate-less lowering
668 // properly handles all type indices by design:
669 markAllIdxsAsCovered();
670 return actionIf(Action: LegalizeAction::Lower, Predicate: always);
671 }
672 /// The instruction is lowered if predicate is true. Keep type index 0 as the
673 /// same type.
674 LegalizeRuleSet &lowerIf(LegalityPredicate Predicate) {
675 using namespace LegalizeMutations;
676 // We have no choice but conservatively assume that lowering with a
677 // free-form user provided Predicate properly handles all type indices:
678 markAllIdxsAsCovered();
679 return actionIf(Action: LegalizeAction::Lower, Predicate);
680 }
681 /// The instruction is lowered if predicate is true.
682 LegalizeRuleSet &lowerIf(LegalityPredicate Predicate,
683 LegalizeMutation Mutation) {
684 // We have no choice but conservatively assume that lowering with a
685 // free-form user provided Predicate properly handles all type indices:
686 markAllIdxsAsCovered();
687 return actionIf(Action: LegalizeAction::Lower, Predicate, Mutation);
688 }
689 /// The instruction is lowered when type index 0 is any type in the given
690 /// list. Keep type index 0 as the same type.
691 LegalizeRuleSet &lowerFor(std::initializer_list<LLT> Types) {
692 return actionFor(Action: LegalizeAction::Lower, Types);
693 }
694 /// The instruction is lowered when type index 0 is any type in the given
695 /// list.
696 LegalizeRuleSet &lowerFor(std::initializer_list<LLT> Types,
697 LegalizeMutation Mutation) {
698 return actionFor(Action: LegalizeAction::Lower, Types, Mutation);
699 }
700 /// The instruction is lowered when type indexes 0 and 1 is any type pair in
701 /// the given list. Keep type index 0 as the same type.
702 LegalizeRuleSet &lowerFor(std::initializer_list<std::pair<LLT, LLT>> Types) {
703 return actionFor(Action: LegalizeAction::Lower, Types);
704 }
705 /// The instruction is lowered when type indexes 0 and 1 is any type pair in
706 /// the given list.
707 LegalizeRuleSet &lowerFor(std::initializer_list<std::pair<LLT, LLT>> Types,
708 LegalizeMutation Mutation) {
709 return actionFor(Action: LegalizeAction::Lower, Types, Mutation);
710 }
711 /// The instruction is lowered when type indexes 0 and 1 are both in their
712 /// respective lists.
713 LegalizeRuleSet &lowerForCartesianProduct(std::initializer_list<LLT> Types0,
714 std::initializer_list<LLT> Types1) {
715 using namespace LegalityPredicates;
716 return actionForCartesianProduct(Action: LegalizeAction::Lower, Types0, Types1);
717 }
718 /// The instruction is lowered when type indexes 0, 1, and 2 are all in
719 /// their respective lists.
720 LegalizeRuleSet &lowerForCartesianProduct(std::initializer_list<LLT> Types0,
721 std::initializer_list<LLT> Types1,
722 std::initializer_list<LLT> Types2) {
723 using namespace LegalityPredicates;
724 return actionForCartesianProduct(Action: LegalizeAction::Lower, Types0, Types1,
725 Types2);
726 }
727
728 /// The instruction is emitted as a library call.
729 LegalizeRuleSet &libcall() {
730 using namespace LegalizeMutations;
731 // We have no choice but conservatively assume that predicate-less lowering
732 // properly handles all type indices by design:
733 markAllIdxsAsCovered();
734 return actionIf(Action: LegalizeAction::Libcall, Predicate: always);
735 }
736
737 /// Like legalIf, but for the Libcall action.
738 LegalizeRuleSet &libcallIf(LegalityPredicate Predicate) {
739 // We have no choice but conservatively assume that a libcall with a
740 // free-form user provided Predicate properly handles all type indices:
741 markAllIdxsAsCovered();
742 return actionIf(Action: LegalizeAction::Libcall, Predicate);
743 }
744 LegalizeRuleSet &libcallFor(std::initializer_list<LLT> Types) {
745 return actionFor(Action: LegalizeAction::Libcall, Types);
746 }
747 LegalizeRuleSet &
748 libcallFor(std::initializer_list<std::pair<LLT, LLT>> Types) {
749 return actionFor(Action: LegalizeAction::Libcall, Types);
750 }
751 LegalizeRuleSet &
752 libcallForCartesianProduct(std::initializer_list<LLT> Types) {
753 return actionForCartesianProduct(Action: LegalizeAction::Libcall, Types);
754 }
755 LegalizeRuleSet &
756 libcallForCartesianProduct(std::initializer_list<LLT> Types0,
757 std::initializer_list<LLT> Types1) {
758 return actionForCartesianProduct(Action: LegalizeAction::Libcall, Types0, Types1);
759 }
760
761 /// Widen the scalar to the one selected by the mutation if the predicate is
762 /// true.
763 LegalizeRuleSet &widenScalarIf(LegalityPredicate Predicate,
764 LegalizeMutation Mutation) {
765 // We have no choice but conservatively assume that an action with a
766 // free-form user provided Predicate properly handles all type indices:
767 markAllIdxsAsCovered();
768 return actionIf(Action: LegalizeAction::WidenScalar, Predicate, Mutation);
769 }
770 /// Narrow the scalar to the one selected by the mutation if the predicate is
771 /// true.
772 LegalizeRuleSet &narrowScalarIf(LegalityPredicate Predicate,
773 LegalizeMutation Mutation) {
774 // We have no choice but conservatively assume that an action with a
775 // free-form user provided Predicate properly handles all type indices:
776 markAllIdxsAsCovered();
777 return actionIf(Action: LegalizeAction::NarrowScalar, Predicate, Mutation);
778 }
779 /// Narrow the scalar, specified in mutation, when type indexes 0 and 1 is any
780 /// type pair in the given list.
781 LegalizeRuleSet &
782 narrowScalarFor(std::initializer_list<std::pair<LLT, LLT>> Types,
783 LegalizeMutation Mutation) {
784 return actionFor(Action: LegalizeAction::NarrowScalar, Types, Mutation);
785 }
786
787 /// Add more elements to reach the type selected by the mutation if the
788 /// predicate is true.
789 LegalizeRuleSet &moreElementsIf(LegalityPredicate Predicate,
790 LegalizeMutation Mutation) {
791 // We have no choice but conservatively assume that an action with a
792 // free-form user provided Predicate properly handles all type indices:
793 markAllIdxsAsCovered();
794 return actionIf(Action: LegalizeAction::MoreElements, Predicate, Mutation);
795 }
796 /// Remove elements to reach the type selected by the mutation if the
797 /// predicate is true.
798 LegalizeRuleSet &fewerElementsIf(LegalityPredicate Predicate,
799 LegalizeMutation Mutation) {
800 // We have no choice but conservatively assume that an action with a
801 // free-form user provided Predicate properly handles all type indices:
802 markAllIdxsAsCovered();
803 return actionIf(Action: LegalizeAction::FewerElements, Predicate, Mutation);
804 }
805
806 /// The instruction is unsupported.
807 LegalizeRuleSet &unsupported() {
808 markAllIdxsAsCovered();
809 return actionIf(Action: LegalizeAction::Unsupported, Predicate: always);
810 }
811 LegalizeRuleSet &unsupportedIf(LegalityPredicate Predicate) {
812 return actionIf(Action: LegalizeAction::Unsupported, Predicate);
813 }
814
815 LegalizeRuleSet &unsupportedFor(std::initializer_list<LLT> Types) {
816 return actionFor(Action: LegalizeAction::Unsupported, Types);
817 }
818
819 LegalizeRuleSet &unsupportedIfMemSizeNotPow2() {
820 return actionIf(Action: LegalizeAction::Unsupported,
821 Predicate: LegalityPredicates::memSizeInBytesNotPow2(MMOIdx: 0));
822 }
823
824 /// Lower a memory operation if the memory size, rounded to bytes, is not a
825 /// power of 2. For example, this will not trigger for s1 or s7, but will for
826 /// s24.
827 LegalizeRuleSet &lowerIfMemSizeNotPow2() {
828 return actionIf(Action: LegalizeAction::Lower,
829 Predicate: LegalityPredicates::memSizeInBytesNotPow2(MMOIdx: 0));
830 }
831
832 /// Lower a memory operation if the memory access size is not a round power of
833 /// 2 byte size. This is stricter than lowerIfMemSizeNotPow2, and more likely
834 /// what you want (e.g. this will lower s1, s7 and s24).
835 LegalizeRuleSet &lowerIfMemSizeNotByteSizePow2() {
836 return actionIf(Action: LegalizeAction::Lower,
837 Predicate: LegalityPredicates::memSizeNotByteSizePow2(MMOIdx: 0));
838 }
839
840 LegalizeRuleSet &customIf(LegalityPredicate Predicate) {
841 // We have no choice but conservatively assume that a custom action with a
842 // free-form user provided Predicate properly handles all type indices:
843 markAllIdxsAsCovered();
844 return actionIf(Action: LegalizeAction::Custom, Predicate);
845 }
846 LegalizeRuleSet &customFor(std::initializer_list<LLT> Types) {
847 return actionFor(Action: LegalizeAction::Custom, Types);
848 }
849
850 /// The instruction is custom when type indexes 0 and 1 is any type pair in the
851 /// given list.
852 LegalizeRuleSet &customFor(std::initializer_list<std::pair<LLT, LLT>> Types) {
853 return actionFor(Action: LegalizeAction::Custom, Types);
854 }
855
856 LegalizeRuleSet &customForCartesianProduct(std::initializer_list<LLT> Types) {
857 return actionForCartesianProduct(Action: LegalizeAction::Custom, Types);
858 }
859 /// The instruction is custom when type indexes 0 and 1 are both in their
860 /// respective lists.
861 LegalizeRuleSet &
862 customForCartesianProduct(std::initializer_list<LLT> Types0,
863 std::initializer_list<LLT> Types1) {
864 return actionForCartesianProduct(Action: LegalizeAction::Custom, Types0, Types1);
865 }
866 /// The instruction is custom when type indexes 0, 1, and 2 are all in
867 /// their respective lists.
868 LegalizeRuleSet &
869 customForCartesianProduct(std::initializer_list<LLT> Types0,
870 std::initializer_list<LLT> Types1,
871 std::initializer_list<LLT> Types2) {
872 return actionForCartesianProduct(Action: LegalizeAction::Custom, Types0, Types1,
873 Types2);
874 }
875
876 /// Unconditionally custom lower.
877 LegalizeRuleSet &custom() {
878 return customIf(Predicate: always);
879 }
880
881 /// Widen the scalar to the next power of two that is at least MinSize.
882 /// No effect if the type is not a scalar or is a power of two.
883 LegalizeRuleSet &widenScalarToNextPow2(unsigned TypeIdx,
884 unsigned MinSize = 0) {
885 using namespace LegalityPredicates;
886 return actionIf(
887 Action: LegalizeAction::WidenScalar, Predicate: sizeNotPow2(TypeIdx: typeIdx(TypeIdx)),
888 Mutation: LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, Min: MinSize));
889 }
890
891 /// Widen the scalar to the next multiple of Size. No effect if the
892 /// type is not a scalar or is a multiple of Size.
893 LegalizeRuleSet &widenScalarToNextMultipleOf(unsigned TypeIdx,
894 unsigned Size) {
895 using namespace LegalityPredicates;
896 return actionIf(
897 Action: LegalizeAction::WidenScalar, Predicate: sizeNotMultipleOf(TypeIdx: typeIdx(TypeIdx), Size),
898 Mutation: LegalizeMutations::widenScalarOrEltToNextMultipleOf(TypeIdx, Size));
899 }
900
901 /// Widen the scalar or vector element type to the next power of two that is
902 /// at least MinSize. No effect if the scalar size is a power of two.
903 LegalizeRuleSet &widenScalarOrEltToNextPow2(unsigned TypeIdx,
904 unsigned MinSize = 0) {
905 using namespace LegalityPredicates;
906 return actionIf(
907 Action: LegalizeAction::WidenScalar, Predicate: scalarOrEltSizeNotPow2(TypeIdx: typeIdx(TypeIdx)),
908 Mutation: LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, Min: MinSize));
909 }
910
911 LegalizeRuleSet &narrowScalar(unsigned TypeIdx, LegalizeMutation Mutation) {
912 using namespace LegalityPredicates;
913 return actionIf(Action: LegalizeAction::NarrowScalar, Predicate: isScalar(TypeIdx: typeIdx(TypeIdx)),
914 Mutation);
915 }
916
917 LegalizeRuleSet &scalarize(unsigned TypeIdx) {
918 using namespace LegalityPredicates;
919 return actionIf(Action: LegalizeAction::FewerElements, Predicate: isVector(TypeIdx: typeIdx(TypeIdx)),
920 Mutation: LegalizeMutations::scalarize(TypeIdx));
921 }
922
923 LegalizeRuleSet &scalarizeIf(LegalityPredicate Predicate, unsigned TypeIdx) {
924 using namespace LegalityPredicates;
925 return actionIf(Action: LegalizeAction::FewerElements,
926 Predicate: all(P0: Predicate, P1: isVector(TypeIdx: typeIdx(TypeIdx))),
927 Mutation: LegalizeMutations::scalarize(TypeIdx));
928 }
929
930 /// Ensure the scalar or element is at least as wide as Ty.
931 LegalizeRuleSet &minScalarOrElt(unsigned TypeIdx, const LLT Ty) {
932 using namespace LegalityPredicates;
933 using namespace LegalizeMutations;
934 return actionIf(Action: LegalizeAction::WidenScalar,
935 Predicate: scalarOrEltNarrowerThan(TypeIdx, Size: Ty.getScalarSizeInBits()),
936 Mutation: changeElementTo(TypeIdx: typeIdx(TypeIdx), Ty));
937 }
938
939 /// Ensure the scalar or element is at least as wide as Ty.
940 LegalizeRuleSet &minScalarOrEltIf(LegalityPredicate Predicate,
941 unsigned TypeIdx, const LLT Ty) {
942 using namespace LegalityPredicates;
943 using namespace LegalizeMutations;
944 return actionIf(Action: LegalizeAction::WidenScalar,
945 Predicate: all(P0: Predicate, P1: scalarOrEltNarrowerThan(
946 TypeIdx, Size: Ty.getScalarSizeInBits())),
947 Mutation: changeElementTo(TypeIdx: typeIdx(TypeIdx), Ty));
948 }
949
950 /// Ensure the vector size is at least as wide as VectorSize by promoting the
951 /// element.
952 LegalizeRuleSet &widenVectorEltsToVectorMinSize(unsigned TypeIdx,
953 unsigned VectorSize) {
954 using namespace LegalityPredicates;
955 using namespace LegalizeMutations;
956 return actionIf(
957 Action: LegalizeAction::WidenScalar,
958 Predicate: [=](const LegalityQuery &Query) {
959 const LLT VecTy = Query.Types[TypeIdx];
960 return VecTy.isVector() && !VecTy.isScalable() &&
961 VecTy.getSizeInBits() < VectorSize;
962 },
963 Mutation: [=](const LegalityQuery &Query) {
964 const LLT VecTy = Query.Types[TypeIdx];
965 unsigned NumElts = VecTy.getNumElements();
966 unsigned MinSize = VectorSize / NumElts;
967 LLT NewTy = LLT::fixed_vector(NumElements: NumElts, ScalarTy: LLT::scalar(SizeInBits: MinSize));
968 return std::make_pair(x: TypeIdx, y&: NewTy);
969 });
970 }
971
972 /// Ensure the scalar is at least as wide as Ty.
973 LegalizeRuleSet &minScalar(unsigned TypeIdx, const LLT Ty) {
974 using namespace LegalityPredicates;
975 using namespace LegalizeMutations;
976 return actionIf(Action: LegalizeAction::WidenScalar,
977 Predicate: scalarNarrowerThan(TypeIdx, Size: Ty.getSizeInBits()),
978 Mutation: changeTo(TypeIdx: typeIdx(TypeIdx), Ty));
979 }
980
981 /// Ensure the scalar is at least as wide as Ty if condition is met.
982 LegalizeRuleSet &minScalarIf(LegalityPredicate Predicate, unsigned TypeIdx,
983 const LLT Ty) {
984 using namespace LegalityPredicates;
985 using namespace LegalizeMutations;
986 return actionIf(
987 Action: LegalizeAction::WidenScalar,
988 Predicate: [=](const LegalityQuery &Query) {
989 const LLT QueryTy = Query.Types[TypeIdx];
990 return QueryTy.isScalar() &&
991 QueryTy.getSizeInBits() < Ty.getSizeInBits() &&
992 Predicate(Query);
993 },
994 Mutation: changeTo(TypeIdx: typeIdx(TypeIdx), Ty));
995 }
996
997 /// Ensure the scalar is at most as wide as Ty.
998 LegalizeRuleSet &maxScalarOrElt(unsigned TypeIdx, const LLT Ty) {
999 using namespace LegalityPredicates;
1000 using namespace LegalizeMutations;
1001 return actionIf(Action: LegalizeAction::NarrowScalar,
1002 Predicate: scalarOrEltWiderThan(TypeIdx, Size: Ty.getScalarSizeInBits()),
1003 Mutation: changeElementTo(TypeIdx: typeIdx(TypeIdx), Ty));
1004 }
1005
1006 /// Ensure the scalar is at most as wide as Ty.
1007 LegalizeRuleSet &maxScalar(unsigned TypeIdx, const LLT Ty) {
1008 using namespace LegalityPredicates;
1009 using namespace LegalizeMutations;
1010 return actionIf(Action: LegalizeAction::NarrowScalar,
1011 Predicate: scalarWiderThan(TypeIdx, Size: Ty.getSizeInBits()),
1012 Mutation: changeTo(TypeIdx: typeIdx(TypeIdx), Ty));
1013 }
1014
1015 /// Conditionally limit the maximum size of the scalar.
1016 /// For example, when the maximum size of one type depends on the size of
1017 /// another such as extracting N bits from an M bit container.
1018 LegalizeRuleSet &maxScalarIf(LegalityPredicate Predicate, unsigned TypeIdx,
1019 const LLT Ty) {
1020 using namespace LegalityPredicates;
1021 using namespace LegalizeMutations;
1022 return actionIf(
1023 Action: LegalizeAction::NarrowScalar,
1024 Predicate: [=](const LegalityQuery &Query) {
1025 const LLT QueryTy = Query.Types[TypeIdx];
1026 return QueryTy.isScalar() &&
1027 QueryTy.getSizeInBits() > Ty.getSizeInBits() &&
1028 Predicate(Query);
1029 },
1030 Mutation: changeElementTo(TypeIdx: typeIdx(TypeIdx), Ty));
1031 }
1032
1033 /// Limit the range of scalar sizes to MinTy and MaxTy.
1034 LegalizeRuleSet &clampScalar(unsigned TypeIdx, const LLT MinTy,
1035 const LLT MaxTy) {
1036 assert(MinTy.isScalar() && MaxTy.isScalar() && "Expected scalar types");
1037 return minScalar(TypeIdx, Ty: MinTy).maxScalar(TypeIdx, Ty: MaxTy);
1038 }
1039
1040 /// Limit the range of scalar sizes to MinTy and MaxTy.
1041 LegalizeRuleSet &clampScalarOrElt(unsigned TypeIdx, const LLT MinTy,
1042 const LLT MaxTy) {
1043 return minScalarOrElt(TypeIdx, Ty: MinTy).maxScalarOrElt(TypeIdx, Ty: MaxTy);
1044 }
1045
1046 /// Widen the scalar to match the size of another.
1047 LegalizeRuleSet &minScalarSameAs(unsigned TypeIdx, unsigned LargeTypeIdx) {
1048 typeIdx(TypeIdx);
1049 return widenScalarIf(
1050 Predicate: [=](const LegalityQuery &Query) {
1051 return Query.Types[LargeTypeIdx].getScalarSizeInBits() >
1052 Query.Types[TypeIdx].getSizeInBits();
1053 },
1054 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx, FromTypeIdx: LargeTypeIdx));
1055 }
1056
1057 /// Narrow the scalar to match the size of another.
1058 LegalizeRuleSet &maxScalarSameAs(unsigned TypeIdx, unsigned NarrowTypeIdx) {
1059 typeIdx(TypeIdx);
1060 return narrowScalarIf(
1061 Predicate: [=](const LegalityQuery &Query) {
1062 return Query.Types[NarrowTypeIdx].getScalarSizeInBits() <
1063 Query.Types[TypeIdx].getSizeInBits();
1064 },
1065 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx, FromTypeIdx: NarrowTypeIdx));
1066 }
1067
1068 /// Change the type \p TypeIdx to have the same scalar size as type \p
1069 /// SameSizeIdx.
1070 LegalizeRuleSet &scalarSameSizeAs(unsigned TypeIdx, unsigned SameSizeIdx) {
1071 return minScalarSameAs(TypeIdx, LargeTypeIdx: SameSizeIdx)
1072 .maxScalarSameAs(TypeIdx, NarrowTypeIdx: SameSizeIdx);
1073 }
1074
1075 /// Conditionally widen the scalar or elt to match the size of another.
1076 LegalizeRuleSet &minScalarEltSameAsIf(LegalityPredicate Predicate,
1077 unsigned TypeIdx, unsigned LargeTypeIdx) {
1078 typeIdx(TypeIdx);
1079 return widenScalarIf(
1080 Predicate: [=](const LegalityQuery &Query) {
1081 return Query.Types[LargeTypeIdx].getScalarSizeInBits() >
1082 Query.Types[TypeIdx].getScalarSizeInBits() &&
1083 Predicate(Query);
1084 },
1085 Mutation: [=](const LegalityQuery &Query) {
1086 LLT T = Query.Types[LargeTypeIdx];
1087 if (T.isPointerVector())
1088 T = T.changeElementType(NewEltTy: LLT::scalar(SizeInBits: T.getScalarSizeInBits()));
1089 return std::make_pair(x: TypeIdx, y&: T);
1090 });
1091 }
1092
1093 /// Conditionally narrow the scalar or elt to match the size of another.
1094 LegalizeRuleSet &maxScalarEltSameAsIf(LegalityPredicate Predicate,
1095 unsigned TypeIdx,
1096 unsigned SmallTypeIdx) {
1097 typeIdx(TypeIdx);
1098 return narrowScalarIf(
1099 Predicate: [=](const LegalityQuery &Query) {
1100 return Query.Types[SmallTypeIdx].getScalarSizeInBits() <
1101 Query.Types[TypeIdx].getScalarSizeInBits() &&
1102 Predicate(Query);
1103 },
1104 Mutation: [=](const LegalityQuery &Query) {
1105 LLT T = Query.Types[SmallTypeIdx];
1106 return std::make_pair(x: TypeIdx, y&: T);
1107 });
1108 }
1109
1110 /// Add more elements to the vector to reach the next power of two.
1111 /// No effect if the type is not a vector or the element count is a power of
1112 /// two.
1113 LegalizeRuleSet &moreElementsToNextPow2(unsigned TypeIdx) {
1114 using namespace LegalityPredicates;
1115 return actionIf(Action: LegalizeAction::MoreElements,
1116 Predicate: numElementsNotPow2(TypeIdx: typeIdx(TypeIdx)),
1117 Mutation: LegalizeMutations::moreElementsToNextPow2(TypeIdx));
1118 }
1119
1120 /// Limit the number of elements in EltTy vectors to at least MinElements.
1121 LegalizeRuleSet &clampMinNumElements(unsigned TypeIdx, const LLT EltTy,
1122 unsigned MinElements) {
1123 // Mark the type index as covered:
1124 typeIdx(TypeIdx);
1125 return actionIf(
1126 Action: LegalizeAction::MoreElements,
1127 Predicate: [=](const LegalityQuery &Query) {
1128 LLT VecTy = Query.Types[TypeIdx];
1129 return VecTy.isVector() && VecTy.getElementType() == EltTy &&
1130 VecTy.getNumElements() < MinElements;
1131 },
1132 Mutation: [=](const LegalityQuery &Query) {
1133 LLT VecTy = Query.Types[TypeIdx];
1134 return std::make_pair(
1135 x: TypeIdx, y: LLT::fixed_vector(NumElements: MinElements, ScalarTy: VecTy.getElementType()));
1136 });
1137 }
1138
1139 /// Set number of elements to nearest larger multiple of NumElts.
1140 LegalizeRuleSet &alignNumElementsTo(unsigned TypeIdx, const LLT EltTy,
1141 unsigned NumElts) {
1142 typeIdx(TypeIdx);
1143 return actionIf(
1144 Action: LegalizeAction::MoreElements,
1145 Predicate: [=](const LegalityQuery &Query) {
1146 LLT VecTy = Query.Types[TypeIdx];
1147 return VecTy.isVector() && VecTy.getElementType() == EltTy &&
1148 (VecTy.getNumElements() % NumElts != 0);
1149 },
1150 Mutation: [=](const LegalityQuery &Query) {
1151 LLT VecTy = Query.Types[TypeIdx];
1152 unsigned NewSize = alignTo(Value: VecTy.getNumElements(), Align: NumElts);
1153 return std::make_pair(
1154 x: TypeIdx, y: LLT::fixed_vector(NumElements: NewSize, ScalarTy: VecTy.getElementType()));
1155 });
1156 }
1157
1158 /// Limit the number of elements in EltTy vectors to at most MaxElements.
1159 LegalizeRuleSet &clampMaxNumElements(unsigned TypeIdx, const LLT EltTy,
1160 unsigned MaxElements) {
1161 // Mark the type index as covered:
1162 typeIdx(TypeIdx);
1163 return actionIf(
1164 Action: LegalizeAction::FewerElements,
1165 Predicate: [=](const LegalityQuery &Query) {
1166 LLT VecTy = Query.Types[TypeIdx];
1167 return VecTy.isVector() && VecTy.getElementType() == EltTy &&
1168 VecTy.getNumElements() > MaxElements;
1169 },
1170 Mutation: [=](const LegalityQuery &Query) {
1171 LLT VecTy = Query.Types[TypeIdx];
1172 LLT NewTy = LLT::scalarOrVector(EC: ElementCount::getFixed(MinVal: MaxElements),
1173 ScalarTy: VecTy.getElementType());
1174 return std::make_pair(x: TypeIdx, y&: NewTy);
1175 });
1176 }
1177 /// Limit the number of elements for the given vectors to at least MinTy's
1178 /// number of elements and at most MaxTy's number of elements.
1179 ///
1180 /// No effect if the type is not a vector or does not have the same element
1181 /// type as the constraints.
1182 /// The element type of MinTy and MaxTy must match.
1183 LegalizeRuleSet &clampNumElements(unsigned TypeIdx, const LLT MinTy,
1184 const LLT MaxTy) {
1185 assert(MinTy.getElementType() == MaxTy.getElementType() &&
1186 "Expected element types to agree");
1187
1188 const LLT EltTy = MinTy.getElementType();
1189 return clampMinNumElements(TypeIdx, EltTy, MinElements: MinTy.getNumElements())
1190 .clampMaxNumElements(TypeIdx, EltTy, MaxElements: MaxTy.getNumElements());
1191 }
1192
1193 /// Express \p EltTy vectors strictly using vectors with \p NumElts elements
1194 /// (or scalars when \p NumElts equals 1).
1195 /// First pad with undef elements to nearest larger multiple of \p NumElts.
1196 /// Then perform split with all sub-instructions having the same type.
1197 /// Using clampMaxNumElements (non-strict) can result in leftover instruction
1198 /// with different type (fewer elements then \p NumElts or scalar).
1199 /// No effect if the type is not a vector.
1200 LegalizeRuleSet &clampMaxNumElementsStrict(unsigned TypeIdx, const LLT EltTy,
1201 unsigned NumElts) {
1202 return alignNumElementsTo(TypeIdx, EltTy, NumElts)
1203 .clampMaxNumElements(TypeIdx, EltTy, MaxElements: NumElts);
1204 }
1205
1206 /// Fallback on the previous implementation. This should only be used while
1207 /// porting a rule.
1208 LegalizeRuleSet &fallback() {
1209 add(Rule: {always, LegalizeAction::UseLegacyRules});
1210 return *this;
1211 }
1212
1213 /// Check if there is no type index which is obviously not handled by the
1214 /// LegalizeRuleSet in any way at all.
1215 /// \pre Type indices of the opcode form a dense [0, \p NumTypeIdxs) set.
1216 bool verifyTypeIdxsCoverage(unsigned NumTypeIdxs) const;
1217 /// Check if there is no imm index which is obviously not handled by the
1218 /// LegalizeRuleSet in any way at all.
1219 /// \pre Type indices of the opcode form a dense [0, \p NumTypeIdxs) set.
1220 bool verifyImmIdxsCoverage(unsigned NumImmIdxs) const;
1221
1222 /// Apply the ruleset to the given LegalityQuery.
1223 LegalizeActionStep apply(const LegalityQuery &Query) const;
1224};
1225
1226class LegalizerInfo {
1227public:
1228 virtual ~LegalizerInfo() = default;
1229
1230 const LegacyLegalizerInfo &getLegacyLegalizerInfo() const {
1231 return LegacyInfo;
1232 }
1233 LegacyLegalizerInfo &getLegacyLegalizerInfo() { return LegacyInfo; }
1234
1235 unsigned getOpcodeIdxForOpcode(unsigned Opcode) const;
1236 unsigned getActionDefinitionsIdx(unsigned Opcode) const;
1237
1238 /// Perform simple self-diagnostic and assert if there is anything obviously
1239 /// wrong with the actions set up.
1240 void verify(const MCInstrInfo &MII) const;
1241
1242 /// Get the action definitions for the given opcode. Use this to run a
1243 /// LegalityQuery through the definitions.
1244 const LegalizeRuleSet &getActionDefinitions(unsigned Opcode) const;
1245
1246 /// Get the action definition builder for the given opcode. Use this to define
1247 /// the action definitions.
1248 ///
1249 /// It is an error to request an opcode that has already been requested by the
1250 /// multiple-opcode variant.
1251 LegalizeRuleSet &getActionDefinitionsBuilder(unsigned Opcode);
1252
1253 /// Get the action definition builder for the given set of opcodes. Use this
1254 /// to define the action definitions for multiple opcodes at once. The first
1255 /// opcode given will be considered the representative opcode and will hold
1256 /// the definitions whereas the other opcodes will be configured to refer to
1257 /// the representative opcode. This lowers memory requirements and very
1258 /// slightly improves performance.
1259 ///
1260 /// It would be very easy to introduce unexpected side-effects as a result of
1261 /// this aliasing if it were permitted to request different but intersecting
1262 /// sets of opcodes but that is difficult to keep track of. It is therefore an
1263 /// error to request the same opcode twice using this API, to request an
1264 /// opcode that already has definitions, or to use the single-opcode API on an
1265 /// opcode that has already been requested by this API.
1266 LegalizeRuleSet &
1267 getActionDefinitionsBuilder(std::initializer_list<unsigned> Opcodes);
1268 void aliasActionDefinitions(unsigned OpcodeTo, unsigned OpcodeFrom);
1269
1270 /// Determine what action should be taken to legalize the described
1271 /// instruction. Requires computeTables to have been called.
1272 ///
1273 /// \returns a description of the next legalization step to perform.
1274 LegalizeActionStep getAction(const LegalityQuery &Query) const;
1275
1276 /// Determine what action should be taken to legalize the given generic
1277 /// instruction.
1278 ///
1279 /// \returns a description of the next legalization step to perform.
1280 LegalizeActionStep getAction(const MachineInstr &MI,
1281 const MachineRegisterInfo &MRI) const;
1282
1283 bool isLegal(const LegalityQuery &Query) const {
1284 return getAction(Query).Action == LegalizeAction::Legal;
1285 }
1286
1287 bool isLegalOrCustom(const LegalityQuery &Query) const {
1288 auto Action = getAction(Query).Action;
1289 return Action == LegalizeAction::Legal || Action == LegalizeAction::Custom;
1290 }
1291
1292 bool isLegal(const MachineInstr &MI, const MachineRegisterInfo &MRI) const;
1293 bool isLegalOrCustom(const MachineInstr &MI,
1294 const MachineRegisterInfo &MRI) const;
1295
1296 /// Called for instructions with the Custom LegalizationAction.
1297 virtual bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI,
1298 LostDebugLocObserver &LocObserver) const {
1299 llvm_unreachable("must implement this if custom action is used");
1300 }
1301
1302 /// \returns true if MI is either legal or has been legalized and false if not
1303 /// legal.
1304 /// Return true if MI is either legal or has been legalized and false
1305 /// if not legal.
1306 virtual bool legalizeIntrinsic(LegalizerHelper &Helper,
1307 MachineInstr &MI) const {
1308 return true;
1309 }
1310
1311 /// Return the opcode (SEXT/ZEXT/ANYEXT) that should be performed while
1312 /// widening a constant of type SmallTy which targets can override.
1313 /// For eg, the DAG does (SmallTy.isByteSized() ? G_SEXT : G_ZEXT) which
1314 /// will be the default.
1315 virtual unsigned getExtOpcodeForWideningConstant(LLT SmallTy) const;
1316
1317private:
1318 static const int FirstOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_START;
1319 static const int LastOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_END;
1320
1321 LegalizeRuleSet RulesForOpcode[LastOp - FirstOp + 1];
1322 LegacyLegalizerInfo LegacyInfo;
1323};
1324
1325#ifndef NDEBUG
1326/// Checks that MIR is fully legal, returns an illegal instruction if it's not,
1327/// nullptr otherwise
1328const MachineInstr *machineFunctionIsIllegal(const MachineFunction &MF);
1329#endif
1330
1331} // end namespace llvm.
1332
1333#endif // LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H
1334

source code of llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h