1 | //===- llvm/CodeGen/GlobalISel/LegalizerInfo.h ------------------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | /// \file |
9 | /// Interface for Targets to specify which operations they can successfully |
10 | /// select and how the others should be expanded most efficiently. |
11 | /// |
12 | //===----------------------------------------------------------------------===// |
13 | |
14 | #ifndef LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H |
15 | #define LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H |
16 | |
17 | #include "llvm/ADT/SmallBitVector.h" |
18 | #include "llvm/ADT/SmallVector.h" |
19 | #include "llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h" |
20 | #include "llvm/CodeGen/MachineMemOperand.h" |
21 | #include "llvm/CodeGen/TargetOpcodes.h" |
22 | #include "llvm/CodeGenTypes/LowLevelType.h" |
23 | #include "llvm/MC/MCInstrDesc.h" |
24 | #include "llvm/Support/AtomicOrdering.h" |
25 | #include "llvm/Support/CommandLine.h" |
26 | #include <cassert> |
27 | #include <cstdint> |
28 | #include <tuple> |
29 | #include <utility> |
30 | |
31 | namespace llvm { |
32 | |
33 | extern cl::opt<bool> DisableGISelLegalityCheck; |
34 | |
35 | class MachineFunction; |
36 | class raw_ostream; |
37 | class LegalizerHelper; |
38 | class LostDebugLocObserver; |
39 | class MachineInstr; |
40 | class MachineRegisterInfo; |
41 | class MCInstrInfo; |
42 | |
43 | namespace LegalizeActions { |
44 | enum LegalizeAction : std::uint8_t { |
45 | /// The operation is expected to be selectable directly by the target, and |
46 | /// no transformation is necessary. |
47 | Legal, |
48 | |
49 | /// The operation should be synthesized from multiple instructions acting on |
50 | /// a narrower scalar base-type. For example a 64-bit add might be |
51 | /// implemented in terms of 32-bit add-with-carry. |
52 | NarrowScalar, |
53 | |
54 | /// The operation should be implemented in terms of a wider scalar |
55 | /// base-type. For example a <2 x s8> add could be implemented as a <2 |
56 | /// x s32> add (ignoring the high bits). |
57 | WidenScalar, |
58 | |
59 | /// The (vector) operation should be implemented by splitting it into |
60 | /// sub-vectors where the operation is legal. For example a <8 x s64> add |
61 | /// might be implemented as 4 separate <2 x s64> adds. There can be a leftover |
62 | /// if there are not enough elements for last sub-vector e.g. <7 x s64> add |
63 | /// will be implemented as 3 separate <2 x s64> adds and one s64 add. Leftover |
64 | /// types can be avoided by doing MoreElements first. |
65 | FewerElements, |
66 | |
67 | /// The (vector) operation should be implemented by widening the input |
68 | /// vector and ignoring the lanes added by doing so. For example <2 x i8> is |
69 | /// rarely legal, but you might perform an <8 x i8> and then only look at |
70 | /// the first two results. |
71 | MoreElements, |
72 | |
73 | /// Perform the operation on a different, but equivalently sized type. |
74 | Bitcast, |
75 | |
76 | /// The operation itself must be expressed in terms of simpler actions on |
77 | /// this target. E.g. a SREM replaced by an SDIV and subtraction. |
78 | Lower, |
79 | |
80 | /// The operation should be implemented as a call to some kind of runtime |
81 | /// support library. For example this usually happens on machines that don't |
82 | /// support floating-point operations natively. |
83 | Libcall, |
84 | |
85 | /// The target wants to do something special with this combination of |
86 | /// operand and type. A callback will be issued when it is needed. |
87 | Custom, |
88 | |
89 | /// This operation is completely unsupported on the target. A programming |
90 | /// error has occurred. |
91 | Unsupported, |
92 | |
93 | /// Sentinel value for when no action was found in the specified table. |
94 | NotFound, |
95 | |
96 | /// Fall back onto the old rules. |
97 | /// TODO: Remove this once we've migrated |
98 | UseLegacyRules, |
99 | }; |
100 | } // end namespace LegalizeActions |
101 | raw_ostream &operator<<(raw_ostream &OS, LegalizeActions::LegalizeAction Action); |
102 | |
103 | using LegalizeActions::LegalizeAction; |
104 | |
105 | /// The LegalityQuery object bundles together all the information that's needed |
106 | /// to decide whether a given operation is legal or not. |
107 | /// For efficiency, it doesn't make a copy of Types so care must be taken not |
108 | /// to free it before using the query. |
109 | struct LegalityQuery { |
110 | unsigned Opcode; |
111 | ArrayRef<LLT> Types; |
112 | |
113 | struct MemDesc { |
114 | LLT MemoryTy; |
115 | uint64_t AlignInBits; |
116 | AtomicOrdering Ordering; |
117 | |
118 | MemDesc() = default; |
119 | MemDesc(LLT MemoryTy, uint64_t AlignInBits, AtomicOrdering Ordering) |
120 | : MemoryTy(MemoryTy), AlignInBits(AlignInBits), Ordering(Ordering) {} |
121 | MemDesc(const MachineMemOperand &MMO) |
122 | : MemoryTy(MMO.getMemoryType()), |
123 | AlignInBits(MMO.getAlign().value() * 8), |
124 | Ordering(MMO.getSuccessOrdering()) {} |
125 | }; |
126 | |
127 | /// Operations which require memory can use this to place requirements on the |
128 | /// memory type for each MMO. |
129 | ArrayRef<MemDesc> MMODescrs; |
130 | |
131 | constexpr LegalityQuery(unsigned Opcode, const ArrayRef<LLT> Types, |
132 | const ArrayRef<MemDesc> MMODescrs) |
133 | : Opcode(Opcode), Types(Types), MMODescrs(MMODescrs) {} |
134 | constexpr LegalityQuery(unsigned Opcode, const ArrayRef<LLT> Types) |
135 | : LegalityQuery(Opcode, Types, {}) {} |
136 | |
137 | raw_ostream &print(raw_ostream &OS) const; |
138 | }; |
139 | |
140 | /// The result of a query. It either indicates a final answer of Legal or |
141 | /// Unsupported or describes an action that must be taken to make an operation |
142 | /// more legal. |
143 | struct LegalizeActionStep { |
144 | /// The action to take or the final answer. |
145 | LegalizeAction Action; |
146 | /// If describing an action, the type index to change. Otherwise zero. |
147 | unsigned TypeIdx; |
148 | /// If describing an action, the new type for TypeIdx. Otherwise LLT{}. |
149 | LLT NewType; |
150 | |
151 | LegalizeActionStep(LegalizeAction Action, unsigned TypeIdx, |
152 | const LLT NewType) |
153 | : Action(Action), TypeIdx(TypeIdx), NewType(NewType) {} |
154 | |
155 | LegalizeActionStep(LegacyLegalizeActionStep Step) |
156 | : TypeIdx(Step.TypeIdx), NewType(Step.NewType) { |
157 | switch (Step.Action) { |
158 | case LegacyLegalizeActions::Legal: |
159 | Action = LegalizeActions::Legal; |
160 | break; |
161 | case LegacyLegalizeActions::NarrowScalar: |
162 | Action = LegalizeActions::NarrowScalar; |
163 | break; |
164 | case LegacyLegalizeActions::WidenScalar: |
165 | Action = LegalizeActions::WidenScalar; |
166 | break; |
167 | case LegacyLegalizeActions::FewerElements: |
168 | Action = LegalizeActions::FewerElements; |
169 | break; |
170 | case LegacyLegalizeActions::MoreElements: |
171 | Action = LegalizeActions::MoreElements; |
172 | break; |
173 | case LegacyLegalizeActions::Bitcast: |
174 | Action = LegalizeActions::Bitcast; |
175 | break; |
176 | case LegacyLegalizeActions::Lower: |
177 | Action = LegalizeActions::Lower; |
178 | break; |
179 | case LegacyLegalizeActions::Libcall: |
180 | Action = LegalizeActions::Libcall; |
181 | break; |
182 | case LegacyLegalizeActions::Custom: |
183 | Action = LegalizeActions::Custom; |
184 | break; |
185 | case LegacyLegalizeActions::Unsupported: |
186 | Action = LegalizeActions::Unsupported; |
187 | break; |
188 | case LegacyLegalizeActions::NotFound: |
189 | Action = LegalizeActions::NotFound; |
190 | break; |
191 | } |
192 | } |
193 | |
194 | bool operator==(const LegalizeActionStep &RHS) const { |
195 | return std::tie(args: Action, args: TypeIdx, args: NewType) == |
196 | std::tie(args: RHS.Action, args: RHS.TypeIdx, args: RHS.NewType); |
197 | } |
198 | }; |
199 | |
200 | using LegalityPredicate = std::function<bool (const LegalityQuery &)>; |
201 | using LegalizeMutation = |
202 | std::function<std::pair<unsigned, LLT>(const LegalityQuery &)>; |
203 | |
204 | namespace LegalityPredicates { |
205 | struct TypePairAndMemDesc { |
206 | LLT Type0; |
207 | LLT Type1; |
208 | LLT MemTy; |
209 | uint64_t Align; |
210 | |
211 | bool operator==(const TypePairAndMemDesc &Other) const { |
212 | return Type0 == Other.Type0 && Type1 == Other.Type1 && |
213 | Align == Other.Align && MemTy == Other.MemTy; |
214 | } |
215 | |
216 | /// \returns true if this memory access is legal with for the access described |
217 | /// by \p Other (The alignment is sufficient for the size and result type). |
218 | bool isCompatible(const TypePairAndMemDesc &Other) const { |
219 | return Type0 == Other.Type0 && Type1 == Other.Type1 && |
220 | Align >= Other.Align && |
221 | // FIXME: This perhaps should be stricter, but the current legality |
222 | // rules are written only considering the size. |
223 | MemTy.getSizeInBits() == Other.MemTy.getSizeInBits(); |
224 | } |
225 | }; |
226 | |
227 | /// True iff P is false. |
228 | template <typename Predicate> Predicate predNot(Predicate P) { |
229 | return [=](const LegalityQuery &Query) { return !P(Query); }; |
230 | } |
231 | |
232 | /// True iff P0 and P1 are true. |
233 | template<typename Predicate> |
234 | Predicate all(Predicate P0, Predicate P1) { |
235 | return [=](const LegalityQuery &Query) { |
236 | return P0(Query) && P1(Query); |
237 | }; |
238 | } |
239 | /// True iff all given predicates are true. |
240 | template<typename Predicate, typename... Args> |
241 | Predicate all(Predicate P0, Predicate P1, Args... args) { |
242 | return all(all(P0, P1), args...); |
243 | } |
244 | |
245 | /// True iff P0 or P1 are true. |
246 | template<typename Predicate> |
247 | Predicate any(Predicate P0, Predicate P1) { |
248 | return [=](const LegalityQuery &Query) { |
249 | return P0(Query) || P1(Query); |
250 | }; |
251 | } |
252 | /// True iff any given predicates are true. |
253 | template<typename Predicate, typename... Args> |
254 | Predicate any(Predicate P0, Predicate P1, Args... args) { |
255 | return any(any(P0, P1), args...); |
256 | } |
257 | |
258 | /// True iff the given type index is the specified type. |
259 | LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit); |
260 | /// True iff the given type index is one of the specified types. |
261 | LegalityPredicate typeInSet(unsigned TypeIdx, |
262 | std::initializer_list<LLT> TypesInit); |
263 | |
264 | /// True iff the given type index is not the specified type. |
265 | inline LegalityPredicate typeIsNot(unsigned TypeIdx, LLT Type) { |
266 | return [=](const LegalityQuery &Query) { |
267 | return Query.Types[TypeIdx] != Type; |
268 | }; |
269 | } |
270 | |
271 | /// True iff the given types for the given pair of type indexes is one of the |
272 | /// specified type pairs. |
273 | LegalityPredicate |
274 | typePairInSet(unsigned TypeIdx0, unsigned TypeIdx1, |
275 | std::initializer_list<std::pair<LLT, LLT>> TypesInit); |
276 | /// True iff the given types for the given pair of type indexes is one of the |
277 | /// specified type pairs. |
278 | LegalityPredicate typePairAndMemDescInSet( |
279 | unsigned TypeIdx0, unsigned TypeIdx1, unsigned MMOIdx, |
280 | std::initializer_list<TypePairAndMemDesc> TypesAndMemDescInit); |
281 | /// True iff the specified type index is a scalar. |
282 | LegalityPredicate isScalar(unsigned TypeIdx); |
283 | /// True iff the specified type index is a vector. |
284 | LegalityPredicate isVector(unsigned TypeIdx); |
285 | /// True iff the specified type index is a pointer (with any address space). |
286 | LegalityPredicate isPointer(unsigned TypeIdx); |
287 | /// True iff the specified type index is a pointer with the specified address |
288 | /// space. |
289 | LegalityPredicate isPointer(unsigned TypeIdx, unsigned AddrSpace); |
290 | |
291 | /// True if the type index is a vector with element type \p EltTy |
292 | LegalityPredicate elementTypeIs(unsigned TypeIdx, LLT EltTy); |
293 | |
294 | /// True iff the specified type index is a scalar that's narrower than the given |
295 | /// size. |
296 | LegalityPredicate scalarNarrowerThan(unsigned TypeIdx, unsigned Size); |
297 | |
298 | /// True iff the specified type index is a scalar that's wider than the given |
299 | /// size. |
300 | LegalityPredicate scalarWiderThan(unsigned TypeIdx, unsigned Size); |
301 | |
302 | /// True iff the specified type index is a scalar or vector with an element type |
303 | /// that's narrower than the given size. |
304 | LegalityPredicate scalarOrEltNarrowerThan(unsigned TypeIdx, unsigned Size); |
305 | |
306 | /// True iff the specified type index is a scalar or a vector with an element |
307 | /// type that's wider than the given size. |
308 | LegalityPredicate scalarOrEltWiderThan(unsigned TypeIdx, unsigned Size); |
309 | |
310 | /// True iff the specified type index is a scalar whose size is not a multiple |
311 | /// of Size. |
312 | LegalityPredicate sizeNotMultipleOf(unsigned TypeIdx, unsigned Size); |
313 | |
314 | /// True iff the specified type index is a scalar whose size is not a power of |
315 | /// 2. |
316 | LegalityPredicate sizeNotPow2(unsigned TypeIdx); |
317 | |
318 | /// True iff the specified type index is a scalar or vector whose element size |
319 | /// is not a power of 2. |
320 | LegalityPredicate scalarOrEltSizeNotPow2(unsigned TypeIdx); |
321 | |
322 | /// True if the total bitwidth of the specified type index is \p Size bits. |
323 | LegalityPredicate sizeIs(unsigned TypeIdx, unsigned Size); |
324 | |
325 | /// True iff the specified type indices are both the same bit size. |
326 | LegalityPredicate sameSize(unsigned TypeIdx0, unsigned TypeIdx1); |
327 | |
328 | /// True iff the first type index has a larger total bit size than second type |
329 | /// index. |
330 | LegalityPredicate largerThan(unsigned TypeIdx0, unsigned TypeIdx1); |
331 | |
332 | /// True iff the first type index has a smaller total bit size than second type |
333 | /// index. |
334 | LegalityPredicate smallerThan(unsigned TypeIdx0, unsigned TypeIdx1); |
335 | |
336 | /// True iff the specified MMO index has a size (rounded to bytes) that is not a |
337 | /// power of 2. |
338 | LegalityPredicate memSizeInBytesNotPow2(unsigned MMOIdx); |
339 | |
340 | /// True iff the specified MMO index has a size that is not an even byte size, |
341 | /// or that even byte size is not a power of 2. |
342 | LegalityPredicate memSizeNotByteSizePow2(unsigned MMOIdx); |
343 | |
344 | /// True iff the specified type index is a vector whose element count is not a |
345 | /// power of 2. |
346 | LegalityPredicate numElementsNotPow2(unsigned TypeIdx); |
347 | /// True iff the specified MMO index has at an atomic ordering of at Ordering or |
348 | /// stronger. |
349 | LegalityPredicate atomicOrderingAtLeastOrStrongerThan(unsigned MMOIdx, |
350 | AtomicOrdering Ordering); |
351 | } // end namespace LegalityPredicates |
352 | |
353 | namespace LegalizeMutations { |
354 | /// Select this specific type for the given type index. |
355 | LegalizeMutation changeTo(unsigned TypeIdx, LLT Ty); |
356 | |
357 | /// Keep the same type as the given type index. |
358 | LegalizeMutation changeTo(unsigned TypeIdx, unsigned FromTypeIdx); |
359 | |
360 | /// Keep the same scalar or element type as the given type index. |
361 | LegalizeMutation changeElementTo(unsigned TypeIdx, unsigned FromTypeIdx); |
362 | |
363 | /// Keep the same scalar or element type as the given type. |
364 | LegalizeMutation changeElementTo(unsigned TypeIdx, LLT Ty); |
365 | |
366 | /// Keep the same scalar or element type as \p TypeIdx, but take the number of |
367 | /// elements from \p FromTypeIdx. |
368 | LegalizeMutation changeElementCountTo(unsigned TypeIdx, unsigned FromTypeIdx); |
369 | |
370 | /// Keep the same scalar or element type as \p TypeIdx, but take the number of |
371 | /// elements from \p Ty. |
372 | LegalizeMutation changeElementCountTo(unsigned TypeIdx, LLT Ty); |
373 | |
374 | /// Change the scalar size or element size to have the same scalar size as type |
375 | /// index \p FromIndex. Unlike changeElementTo, this discards pointer types and |
376 | /// only changes the size. |
377 | LegalizeMutation changeElementSizeTo(unsigned TypeIdx, unsigned FromTypeIdx); |
378 | |
379 | /// Widen the scalar type or vector element type for the given type index to the |
380 | /// next power of 2. |
381 | LegalizeMutation widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned Min = 0); |
382 | |
383 | /// Widen the scalar type or vector element type for the given type index to |
384 | /// next multiple of \p Size. |
385 | LegalizeMutation widenScalarOrEltToNextMultipleOf(unsigned TypeIdx, |
386 | unsigned Size); |
387 | |
388 | /// Add more elements to the type for the given type index to the next power of |
389 | /// 2. |
390 | LegalizeMutation moreElementsToNextPow2(unsigned TypeIdx, unsigned Min = 0); |
391 | /// Break up the vector type for the given type index into the element type. |
392 | LegalizeMutation scalarize(unsigned TypeIdx); |
393 | } // end namespace LegalizeMutations |
394 | |
395 | /// A single rule in a legalizer info ruleset. |
396 | /// The specified action is chosen when the predicate is true. Where appropriate |
397 | /// for the action (e.g. for WidenScalar) the new type is selected using the |
398 | /// given mutator. |
399 | class LegalizeRule { |
400 | LegalityPredicate Predicate; |
401 | LegalizeAction Action; |
402 | LegalizeMutation Mutation; |
403 | |
404 | public: |
405 | LegalizeRule(LegalityPredicate Predicate, LegalizeAction Action, |
406 | LegalizeMutation Mutation = nullptr) |
407 | : Predicate(Predicate), Action(Action), Mutation(Mutation) {} |
408 | |
409 | /// Test whether the LegalityQuery matches. |
410 | bool match(const LegalityQuery &Query) const { |
411 | return Predicate(Query); |
412 | } |
413 | |
414 | LegalizeAction getAction() const { return Action; } |
415 | |
416 | /// Determine the change to make. |
417 | std::pair<unsigned, LLT> determineMutation(const LegalityQuery &Query) const { |
418 | if (Mutation) |
419 | return Mutation(Query); |
420 | return std::make_pair(x: 0, y: LLT{}); |
421 | } |
422 | }; |
423 | |
424 | class LegalizeRuleSet { |
425 | /// When non-zero, the opcode we are an alias of |
426 | unsigned AliasOf = 0; |
427 | /// If true, there is another opcode that aliases this one |
428 | bool IsAliasedByAnother = false; |
429 | SmallVector<LegalizeRule, 2> Rules; |
430 | |
431 | #ifndef NDEBUG |
432 | /// If bit I is set, this rule set contains a rule that may handle (predicate |
433 | /// or perform an action upon (or both)) the type index I. The uncertainty |
434 | /// comes from free-form rules executing user-provided lambda functions. We |
435 | /// conservatively assume such rules do the right thing and cover all type |
436 | /// indices. The bitset is intentionally 1 bit wider than it absolutely needs |
437 | /// to be to distinguish such cases from the cases where all type indices are |
438 | /// individually handled. |
439 | SmallBitVector TypeIdxsCovered{MCOI::OPERAND_LAST_GENERIC - |
440 | MCOI::OPERAND_FIRST_GENERIC + 2}; |
441 | SmallBitVector ImmIdxsCovered{MCOI::OPERAND_LAST_GENERIC_IMM - |
442 | MCOI::OPERAND_FIRST_GENERIC_IMM + 2}; |
443 | #endif |
444 | |
445 | unsigned typeIdx(unsigned TypeIdx) { |
446 | assert(TypeIdx <= |
447 | (MCOI::OPERAND_LAST_GENERIC - MCOI::OPERAND_FIRST_GENERIC) && |
448 | "Type Index is out of bounds" ); |
449 | #ifndef NDEBUG |
450 | TypeIdxsCovered.set(TypeIdx); |
451 | #endif |
452 | return TypeIdx; |
453 | } |
454 | |
455 | void markAllIdxsAsCovered() { |
456 | #ifndef NDEBUG |
457 | TypeIdxsCovered.set(); |
458 | ImmIdxsCovered.set(); |
459 | #endif |
460 | } |
461 | |
462 | void add(const LegalizeRule &Rule) { |
463 | assert(AliasOf == 0 && |
464 | "RuleSet is aliased, change the representative opcode instead" ); |
465 | Rules.push_back(Elt: Rule); |
466 | } |
467 | |
468 | static bool always(const LegalityQuery &) { return true; } |
469 | |
470 | /// Use the given action when the predicate is true. |
471 | /// Action should not be an action that requires mutation. |
472 | LegalizeRuleSet &actionIf(LegalizeAction Action, |
473 | LegalityPredicate Predicate) { |
474 | add(Rule: {Predicate, Action}); |
475 | return *this; |
476 | } |
477 | /// Use the given action when the predicate is true. |
478 | /// Action should be an action that requires mutation. |
479 | LegalizeRuleSet &actionIf(LegalizeAction Action, LegalityPredicate Predicate, |
480 | LegalizeMutation Mutation) { |
481 | add(Rule: {Predicate, Action, Mutation}); |
482 | return *this; |
483 | } |
484 | /// Use the given action when type index 0 is any type in the given list. |
485 | /// Action should not be an action that requires mutation. |
486 | LegalizeRuleSet &actionFor(LegalizeAction Action, |
487 | std::initializer_list<LLT> Types) { |
488 | using namespace LegalityPredicates; |
489 | return actionIf(Action, Predicate: typeInSet(TypeIdx: typeIdx(TypeIdx: 0), TypesInit: Types)); |
490 | } |
491 | /// Use the given action when type index 0 is any type in the given list. |
492 | /// Action should be an action that requires mutation. |
493 | LegalizeRuleSet &actionFor(LegalizeAction Action, |
494 | std::initializer_list<LLT> Types, |
495 | LegalizeMutation Mutation) { |
496 | using namespace LegalityPredicates; |
497 | return actionIf(Action, Predicate: typeInSet(TypeIdx: typeIdx(TypeIdx: 0), TypesInit: Types), Mutation); |
498 | } |
499 | /// Use the given action when type indexes 0 and 1 is any type pair in the |
500 | /// given list. |
501 | /// Action should not be an action that requires mutation. |
502 | LegalizeRuleSet &actionFor(LegalizeAction Action, |
503 | std::initializer_list<std::pair<LLT, LLT>> Types) { |
504 | using namespace LegalityPredicates; |
505 | return actionIf(Action, Predicate: typePairInSet(TypeIdx0: typeIdx(TypeIdx: 0), TypeIdx1: typeIdx(TypeIdx: 1), TypesInit: Types)); |
506 | } |
507 | /// Use the given action when type indexes 0 and 1 is any type pair in the |
508 | /// given list. |
509 | /// Action should be an action that requires mutation. |
510 | LegalizeRuleSet &actionFor(LegalizeAction Action, |
511 | std::initializer_list<std::pair<LLT, LLT>> Types, |
512 | LegalizeMutation Mutation) { |
513 | using namespace LegalityPredicates; |
514 | return actionIf(Action, Predicate: typePairInSet(TypeIdx0: typeIdx(TypeIdx: 0), TypeIdx1: typeIdx(TypeIdx: 1), TypesInit: Types), |
515 | Mutation); |
516 | } |
517 | /// Use the given action when type index 0 is any type in the given list and |
518 | /// imm index 0 is anything. Action should not be an action that requires |
519 | /// mutation. |
520 | LegalizeRuleSet &actionForTypeWithAnyImm(LegalizeAction Action, |
521 | std::initializer_list<LLT> Types) { |
522 | using namespace LegalityPredicates; |
523 | immIdx(ImmIdx: 0); // Inform verifier imm idx 0 is handled. |
524 | return actionIf(Action, Predicate: typeInSet(TypeIdx: typeIdx(TypeIdx: 0), TypesInit: Types)); |
525 | } |
526 | |
527 | LegalizeRuleSet &actionForTypeWithAnyImm( |
528 | LegalizeAction Action, std::initializer_list<std::pair<LLT, LLT>> Types) { |
529 | using namespace LegalityPredicates; |
530 | immIdx(ImmIdx: 0); // Inform verifier imm idx 0 is handled. |
531 | return actionIf(Action, Predicate: typePairInSet(TypeIdx0: typeIdx(TypeIdx: 0), TypeIdx1: typeIdx(TypeIdx: 1), TypesInit: Types)); |
532 | } |
533 | |
534 | /// Use the given action when type indexes 0 and 1 are both in the given list. |
535 | /// That is, the type pair is in the cartesian product of the list. |
536 | /// Action should not be an action that requires mutation. |
537 | LegalizeRuleSet &actionForCartesianProduct(LegalizeAction Action, |
538 | std::initializer_list<LLT> Types) { |
539 | using namespace LegalityPredicates; |
540 | return actionIf(Action, Predicate: all(P0: typeInSet(TypeIdx: typeIdx(TypeIdx: 0), TypesInit: Types), |
541 | P1: typeInSet(TypeIdx: typeIdx(TypeIdx: 1), TypesInit: Types))); |
542 | } |
543 | /// Use the given action when type indexes 0 and 1 are both in their |
544 | /// respective lists. |
545 | /// That is, the type pair is in the cartesian product of the lists |
546 | /// Action should not be an action that requires mutation. |
547 | LegalizeRuleSet & |
548 | actionForCartesianProduct(LegalizeAction Action, |
549 | std::initializer_list<LLT> Types0, |
550 | std::initializer_list<LLT> Types1) { |
551 | using namespace LegalityPredicates; |
552 | return actionIf(Action, Predicate: all(P0: typeInSet(TypeIdx: typeIdx(TypeIdx: 0), TypesInit: Types0), |
553 | P1: typeInSet(TypeIdx: typeIdx(TypeIdx: 1), TypesInit: Types1))); |
554 | } |
555 | /// Use the given action when type indexes 0, 1, and 2 are all in their |
556 | /// respective lists. |
557 | /// That is, the type triple is in the cartesian product of the lists |
558 | /// Action should not be an action that requires mutation. |
559 | LegalizeRuleSet &actionForCartesianProduct( |
560 | LegalizeAction Action, std::initializer_list<LLT> Types0, |
561 | std::initializer_list<LLT> Types1, std::initializer_list<LLT> Types2) { |
562 | using namespace LegalityPredicates; |
563 | return actionIf(Action, Predicate: all(P0: typeInSet(TypeIdx: typeIdx(TypeIdx: 0), TypesInit: Types0), |
564 | P1: all(P0: typeInSet(TypeIdx: typeIdx(TypeIdx: 1), TypesInit: Types1), |
565 | P1: typeInSet(TypeIdx: typeIdx(TypeIdx: 2), TypesInit: Types2)))); |
566 | } |
567 | |
568 | public: |
569 | LegalizeRuleSet() = default; |
570 | |
571 | bool isAliasedByAnother() { return IsAliasedByAnother; } |
572 | void setIsAliasedByAnother() { IsAliasedByAnother = true; } |
573 | void aliasTo(unsigned Opcode) { |
574 | assert((AliasOf == 0 || AliasOf == Opcode) && |
575 | "Opcode is already aliased to another opcode" ); |
576 | assert(Rules.empty() && "Aliasing will discard rules" ); |
577 | AliasOf = Opcode; |
578 | } |
579 | unsigned getAlias() const { return AliasOf; } |
580 | |
581 | unsigned immIdx(unsigned ImmIdx) { |
582 | assert(ImmIdx <= (MCOI::OPERAND_LAST_GENERIC_IMM - |
583 | MCOI::OPERAND_FIRST_GENERIC_IMM) && |
584 | "Imm Index is out of bounds" ); |
585 | #ifndef NDEBUG |
586 | ImmIdxsCovered.set(ImmIdx); |
587 | #endif |
588 | return ImmIdx; |
589 | } |
590 | |
591 | /// The instruction is legal if predicate is true. |
592 | LegalizeRuleSet &legalIf(LegalityPredicate Predicate) { |
593 | // We have no choice but conservatively assume that the free-form |
594 | // user-provided Predicate properly handles all type indices: |
595 | markAllIdxsAsCovered(); |
596 | return actionIf(Action: LegalizeAction::Legal, Predicate); |
597 | } |
598 | /// The instruction is legal when type index 0 is any type in the given list. |
599 | LegalizeRuleSet &legalFor(std::initializer_list<LLT> Types) { |
600 | return actionFor(Action: LegalizeAction::Legal, Types); |
601 | } |
602 | /// The instruction is legal when type indexes 0 and 1 is any type pair in the |
603 | /// given list. |
604 | LegalizeRuleSet &legalFor(std::initializer_list<std::pair<LLT, LLT>> Types) { |
605 | return actionFor(Action: LegalizeAction::Legal, Types); |
606 | } |
607 | /// The instruction is legal when type index 0 is any type in the given list |
608 | /// and imm index 0 is anything. |
609 | LegalizeRuleSet &legalForTypeWithAnyImm(std::initializer_list<LLT> Types) { |
610 | markAllIdxsAsCovered(); |
611 | return actionForTypeWithAnyImm(Action: LegalizeAction::Legal, Types); |
612 | } |
613 | |
614 | LegalizeRuleSet &legalForTypeWithAnyImm( |
615 | std::initializer_list<std::pair<LLT, LLT>> Types) { |
616 | markAllIdxsAsCovered(); |
617 | return actionForTypeWithAnyImm(Action: LegalizeAction::Legal, Types); |
618 | } |
619 | |
620 | /// The instruction is legal when type indexes 0 and 1 along with the memory |
621 | /// size and minimum alignment is any type and size tuple in the given list. |
622 | LegalizeRuleSet &legalForTypesWithMemDesc( |
623 | std::initializer_list<LegalityPredicates::TypePairAndMemDesc> |
624 | TypesAndMemDesc) { |
625 | return actionIf(Action: LegalizeAction::Legal, |
626 | Predicate: LegalityPredicates::typePairAndMemDescInSet( |
627 | TypeIdx0: typeIdx(TypeIdx: 0), TypeIdx1: typeIdx(TypeIdx: 1), /*MMOIdx*/ MMOIdx: 0, TypesAndMemDescInit: TypesAndMemDesc)); |
628 | } |
629 | /// The instruction is legal when type indexes 0 and 1 are both in the given |
630 | /// list. That is, the type pair is in the cartesian product of the list. |
631 | LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types) { |
632 | return actionForCartesianProduct(Action: LegalizeAction::Legal, Types); |
633 | } |
634 | /// The instruction is legal when type indexes 0 and 1 are both their |
635 | /// respective lists. |
636 | LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types0, |
637 | std::initializer_list<LLT> Types1) { |
638 | return actionForCartesianProduct(Action: LegalizeAction::Legal, Types0, Types1); |
639 | } |
640 | /// The instruction is legal when type indexes 0, 1, and 2 are both their |
641 | /// respective lists. |
642 | LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types0, |
643 | std::initializer_list<LLT> Types1, |
644 | std::initializer_list<LLT> Types2) { |
645 | return actionForCartesianProduct(Action: LegalizeAction::Legal, Types0, Types1, |
646 | Types2); |
647 | } |
648 | |
649 | LegalizeRuleSet &alwaysLegal() { |
650 | using namespace LegalizeMutations; |
651 | markAllIdxsAsCovered(); |
652 | return actionIf(Action: LegalizeAction::Legal, Predicate: always); |
653 | } |
654 | |
655 | /// The specified type index is coerced if predicate is true. |
656 | LegalizeRuleSet &bitcastIf(LegalityPredicate Predicate, |
657 | LegalizeMutation Mutation) { |
658 | // We have no choice but conservatively assume that lowering with a |
659 | // free-form user provided Predicate properly handles all type indices: |
660 | markAllIdxsAsCovered(); |
661 | return actionIf(Action: LegalizeAction::Bitcast, Predicate, Mutation); |
662 | } |
663 | |
664 | /// The instruction is lowered. |
665 | LegalizeRuleSet &lower() { |
666 | using namespace LegalizeMutations; |
667 | // We have no choice but conservatively assume that predicate-less lowering |
668 | // properly handles all type indices by design: |
669 | markAllIdxsAsCovered(); |
670 | return actionIf(Action: LegalizeAction::Lower, Predicate: always); |
671 | } |
672 | /// The instruction is lowered if predicate is true. Keep type index 0 as the |
673 | /// same type. |
674 | LegalizeRuleSet &lowerIf(LegalityPredicate Predicate) { |
675 | using namespace LegalizeMutations; |
676 | // We have no choice but conservatively assume that lowering with a |
677 | // free-form user provided Predicate properly handles all type indices: |
678 | markAllIdxsAsCovered(); |
679 | return actionIf(Action: LegalizeAction::Lower, Predicate); |
680 | } |
681 | /// The instruction is lowered if predicate is true. |
682 | LegalizeRuleSet &lowerIf(LegalityPredicate Predicate, |
683 | LegalizeMutation Mutation) { |
684 | // We have no choice but conservatively assume that lowering with a |
685 | // free-form user provided Predicate properly handles all type indices: |
686 | markAllIdxsAsCovered(); |
687 | return actionIf(Action: LegalizeAction::Lower, Predicate, Mutation); |
688 | } |
689 | /// The instruction is lowered when type index 0 is any type in the given |
690 | /// list. Keep type index 0 as the same type. |
691 | LegalizeRuleSet &lowerFor(std::initializer_list<LLT> Types) { |
692 | return actionFor(Action: LegalizeAction::Lower, Types); |
693 | } |
694 | /// The instruction is lowered when type index 0 is any type in the given |
695 | /// list. |
696 | LegalizeRuleSet &lowerFor(std::initializer_list<LLT> Types, |
697 | LegalizeMutation Mutation) { |
698 | return actionFor(Action: LegalizeAction::Lower, Types, Mutation); |
699 | } |
700 | /// The instruction is lowered when type indexes 0 and 1 is any type pair in |
701 | /// the given list. Keep type index 0 as the same type. |
702 | LegalizeRuleSet &lowerFor(std::initializer_list<std::pair<LLT, LLT>> Types) { |
703 | return actionFor(Action: LegalizeAction::Lower, Types); |
704 | } |
705 | /// The instruction is lowered when type indexes 0 and 1 is any type pair in |
706 | /// the given list. |
707 | LegalizeRuleSet &lowerFor(std::initializer_list<std::pair<LLT, LLT>> Types, |
708 | LegalizeMutation Mutation) { |
709 | return actionFor(Action: LegalizeAction::Lower, Types, Mutation); |
710 | } |
711 | /// The instruction is lowered when type indexes 0 and 1 are both in their |
712 | /// respective lists. |
713 | LegalizeRuleSet &lowerForCartesianProduct(std::initializer_list<LLT> Types0, |
714 | std::initializer_list<LLT> Types1) { |
715 | using namespace LegalityPredicates; |
716 | return actionForCartesianProduct(Action: LegalizeAction::Lower, Types0, Types1); |
717 | } |
718 | /// The instruction is lowered when type indexes 0, 1, and 2 are all in |
719 | /// their respective lists. |
720 | LegalizeRuleSet &lowerForCartesianProduct(std::initializer_list<LLT> Types0, |
721 | std::initializer_list<LLT> Types1, |
722 | std::initializer_list<LLT> Types2) { |
723 | using namespace LegalityPredicates; |
724 | return actionForCartesianProduct(Action: LegalizeAction::Lower, Types0, Types1, |
725 | Types2); |
726 | } |
727 | |
728 | /// The instruction is emitted as a library call. |
729 | LegalizeRuleSet &libcall() { |
730 | using namespace LegalizeMutations; |
731 | // We have no choice but conservatively assume that predicate-less lowering |
732 | // properly handles all type indices by design: |
733 | markAllIdxsAsCovered(); |
734 | return actionIf(Action: LegalizeAction::Libcall, Predicate: always); |
735 | } |
736 | |
737 | /// Like legalIf, but for the Libcall action. |
738 | LegalizeRuleSet &libcallIf(LegalityPredicate Predicate) { |
739 | // We have no choice but conservatively assume that a libcall with a |
740 | // free-form user provided Predicate properly handles all type indices: |
741 | markAllIdxsAsCovered(); |
742 | return actionIf(Action: LegalizeAction::Libcall, Predicate); |
743 | } |
744 | LegalizeRuleSet &libcallFor(std::initializer_list<LLT> Types) { |
745 | return actionFor(Action: LegalizeAction::Libcall, Types); |
746 | } |
747 | LegalizeRuleSet & |
748 | libcallFor(std::initializer_list<std::pair<LLT, LLT>> Types) { |
749 | return actionFor(Action: LegalizeAction::Libcall, Types); |
750 | } |
751 | LegalizeRuleSet & |
752 | libcallForCartesianProduct(std::initializer_list<LLT> Types) { |
753 | return actionForCartesianProduct(Action: LegalizeAction::Libcall, Types); |
754 | } |
755 | LegalizeRuleSet & |
756 | libcallForCartesianProduct(std::initializer_list<LLT> Types0, |
757 | std::initializer_list<LLT> Types1) { |
758 | return actionForCartesianProduct(Action: LegalizeAction::Libcall, Types0, Types1); |
759 | } |
760 | |
761 | /// Widen the scalar to the one selected by the mutation if the predicate is |
762 | /// true. |
763 | LegalizeRuleSet &widenScalarIf(LegalityPredicate Predicate, |
764 | LegalizeMutation Mutation) { |
765 | // We have no choice but conservatively assume that an action with a |
766 | // free-form user provided Predicate properly handles all type indices: |
767 | markAllIdxsAsCovered(); |
768 | return actionIf(Action: LegalizeAction::WidenScalar, Predicate, Mutation); |
769 | } |
770 | /// Narrow the scalar to the one selected by the mutation if the predicate is |
771 | /// true. |
772 | LegalizeRuleSet &narrowScalarIf(LegalityPredicate Predicate, |
773 | LegalizeMutation Mutation) { |
774 | // We have no choice but conservatively assume that an action with a |
775 | // free-form user provided Predicate properly handles all type indices: |
776 | markAllIdxsAsCovered(); |
777 | return actionIf(Action: LegalizeAction::NarrowScalar, Predicate, Mutation); |
778 | } |
779 | /// Narrow the scalar, specified in mutation, when type indexes 0 and 1 is any |
780 | /// type pair in the given list. |
781 | LegalizeRuleSet & |
782 | narrowScalarFor(std::initializer_list<std::pair<LLT, LLT>> Types, |
783 | LegalizeMutation Mutation) { |
784 | return actionFor(Action: LegalizeAction::NarrowScalar, Types, Mutation); |
785 | } |
786 | |
787 | /// Add more elements to reach the type selected by the mutation if the |
788 | /// predicate is true. |
789 | LegalizeRuleSet &moreElementsIf(LegalityPredicate Predicate, |
790 | LegalizeMutation Mutation) { |
791 | // We have no choice but conservatively assume that an action with a |
792 | // free-form user provided Predicate properly handles all type indices: |
793 | markAllIdxsAsCovered(); |
794 | return actionIf(Action: LegalizeAction::MoreElements, Predicate, Mutation); |
795 | } |
796 | /// Remove elements to reach the type selected by the mutation if the |
797 | /// predicate is true. |
798 | LegalizeRuleSet &fewerElementsIf(LegalityPredicate Predicate, |
799 | LegalizeMutation Mutation) { |
800 | // We have no choice but conservatively assume that an action with a |
801 | // free-form user provided Predicate properly handles all type indices: |
802 | markAllIdxsAsCovered(); |
803 | return actionIf(Action: LegalizeAction::FewerElements, Predicate, Mutation); |
804 | } |
805 | |
806 | /// The instruction is unsupported. |
807 | LegalizeRuleSet &unsupported() { |
808 | markAllIdxsAsCovered(); |
809 | return actionIf(Action: LegalizeAction::Unsupported, Predicate: always); |
810 | } |
811 | LegalizeRuleSet &unsupportedIf(LegalityPredicate Predicate) { |
812 | return actionIf(Action: LegalizeAction::Unsupported, Predicate); |
813 | } |
814 | |
815 | LegalizeRuleSet &unsupportedFor(std::initializer_list<LLT> Types) { |
816 | return actionFor(Action: LegalizeAction::Unsupported, Types); |
817 | } |
818 | |
819 | LegalizeRuleSet &unsupportedIfMemSizeNotPow2() { |
820 | return actionIf(Action: LegalizeAction::Unsupported, |
821 | Predicate: LegalityPredicates::memSizeInBytesNotPow2(MMOIdx: 0)); |
822 | } |
823 | |
824 | /// Lower a memory operation if the memory size, rounded to bytes, is not a |
825 | /// power of 2. For example, this will not trigger for s1 or s7, but will for |
826 | /// s24. |
827 | LegalizeRuleSet &lowerIfMemSizeNotPow2() { |
828 | return actionIf(Action: LegalizeAction::Lower, |
829 | Predicate: LegalityPredicates::memSizeInBytesNotPow2(MMOIdx: 0)); |
830 | } |
831 | |
832 | /// Lower a memory operation if the memory access size is not a round power of |
833 | /// 2 byte size. This is stricter than lowerIfMemSizeNotPow2, and more likely |
834 | /// what you want (e.g. this will lower s1, s7 and s24). |
835 | LegalizeRuleSet &lowerIfMemSizeNotByteSizePow2() { |
836 | return actionIf(Action: LegalizeAction::Lower, |
837 | Predicate: LegalityPredicates::memSizeNotByteSizePow2(MMOIdx: 0)); |
838 | } |
839 | |
840 | LegalizeRuleSet &customIf(LegalityPredicate Predicate) { |
841 | // We have no choice but conservatively assume that a custom action with a |
842 | // free-form user provided Predicate properly handles all type indices: |
843 | markAllIdxsAsCovered(); |
844 | return actionIf(Action: LegalizeAction::Custom, Predicate); |
845 | } |
846 | LegalizeRuleSet &customFor(std::initializer_list<LLT> Types) { |
847 | return actionFor(Action: LegalizeAction::Custom, Types); |
848 | } |
849 | |
850 | /// The instruction is custom when type indexes 0 and 1 is any type pair in the |
851 | /// given list. |
852 | LegalizeRuleSet &customFor(std::initializer_list<std::pair<LLT, LLT>> Types) { |
853 | return actionFor(Action: LegalizeAction::Custom, Types); |
854 | } |
855 | |
856 | LegalizeRuleSet &customForCartesianProduct(std::initializer_list<LLT> Types) { |
857 | return actionForCartesianProduct(Action: LegalizeAction::Custom, Types); |
858 | } |
859 | /// The instruction is custom when type indexes 0 and 1 are both in their |
860 | /// respective lists. |
861 | LegalizeRuleSet & |
862 | customForCartesianProduct(std::initializer_list<LLT> Types0, |
863 | std::initializer_list<LLT> Types1) { |
864 | return actionForCartesianProduct(Action: LegalizeAction::Custom, Types0, Types1); |
865 | } |
866 | /// The instruction is custom when type indexes 0, 1, and 2 are all in |
867 | /// their respective lists. |
868 | LegalizeRuleSet & |
869 | customForCartesianProduct(std::initializer_list<LLT> Types0, |
870 | std::initializer_list<LLT> Types1, |
871 | std::initializer_list<LLT> Types2) { |
872 | return actionForCartesianProduct(Action: LegalizeAction::Custom, Types0, Types1, |
873 | Types2); |
874 | } |
875 | |
876 | /// Unconditionally custom lower. |
877 | LegalizeRuleSet &custom() { |
878 | return customIf(Predicate: always); |
879 | } |
880 | |
881 | /// Widen the scalar to the next power of two that is at least MinSize. |
882 | /// No effect if the type is not a scalar or is a power of two. |
883 | LegalizeRuleSet &widenScalarToNextPow2(unsigned TypeIdx, |
884 | unsigned MinSize = 0) { |
885 | using namespace LegalityPredicates; |
886 | return actionIf( |
887 | Action: LegalizeAction::WidenScalar, Predicate: sizeNotPow2(TypeIdx: typeIdx(TypeIdx)), |
888 | Mutation: LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, Min: MinSize)); |
889 | } |
890 | |
891 | /// Widen the scalar to the next multiple of Size. No effect if the |
892 | /// type is not a scalar or is a multiple of Size. |
893 | LegalizeRuleSet &widenScalarToNextMultipleOf(unsigned TypeIdx, |
894 | unsigned Size) { |
895 | using namespace LegalityPredicates; |
896 | return actionIf( |
897 | Action: LegalizeAction::WidenScalar, Predicate: sizeNotMultipleOf(TypeIdx: typeIdx(TypeIdx), Size), |
898 | Mutation: LegalizeMutations::widenScalarOrEltToNextMultipleOf(TypeIdx, Size)); |
899 | } |
900 | |
901 | /// Widen the scalar or vector element type to the next power of two that is |
902 | /// at least MinSize. No effect if the scalar size is a power of two. |
903 | LegalizeRuleSet &widenScalarOrEltToNextPow2(unsigned TypeIdx, |
904 | unsigned MinSize = 0) { |
905 | using namespace LegalityPredicates; |
906 | return actionIf( |
907 | Action: LegalizeAction::WidenScalar, Predicate: scalarOrEltSizeNotPow2(TypeIdx: typeIdx(TypeIdx)), |
908 | Mutation: LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, Min: MinSize)); |
909 | } |
910 | |
911 | LegalizeRuleSet &narrowScalar(unsigned TypeIdx, LegalizeMutation Mutation) { |
912 | using namespace LegalityPredicates; |
913 | return actionIf(Action: LegalizeAction::NarrowScalar, Predicate: isScalar(TypeIdx: typeIdx(TypeIdx)), |
914 | Mutation); |
915 | } |
916 | |
917 | LegalizeRuleSet &scalarize(unsigned TypeIdx) { |
918 | using namespace LegalityPredicates; |
919 | return actionIf(Action: LegalizeAction::FewerElements, Predicate: isVector(TypeIdx: typeIdx(TypeIdx)), |
920 | Mutation: LegalizeMutations::scalarize(TypeIdx)); |
921 | } |
922 | |
923 | LegalizeRuleSet &scalarizeIf(LegalityPredicate Predicate, unsigned TypeIdx) { |
924 | using namespace LegalityPredicates; |
925 | return actionIf(Action: LegalizeAction::FewerElements, |
926 | Predicate: all(P0: Predicate, P1: isVector(TypeIdx: typeIdx(TypeIdx))), |
927 | Mutation: LegalizeMutations::scalarize(TypeIdx)); |
928 | } |
929 | |
930 | /// Ensure the scalar or element is at least as wide as Ty. |
931 | LegalizeRuleSet &minScalarOrElt(unsigned TypeIdx, const LLT Ty) { |
932 | using namespace LegalityPredicates; |
933 | using namespace LegalizeMutations; |
934 | return actionIf(Action: LegalizeAction::WidenScalar, |
935 | Predicate: scalarOrEltNarrowerThan(TypeIdx, Size: Ty.getScalarSizeInBits()), |
936 | Mutation: changeElementTo(TypeIdx: typeIdx(TypeIdx), Ty)); |
937 | } |
938 | |
939 | /// Ensure the scalar or element is at least as wide as Ty. |
940 | LegalizeRuleSet &minScalarOrEltIf(LegalityPredicate Predicate, |
941 | unsigned TypeIdx, const LLT Ty) { |
942 | using namespace LegalityPredicates; |
943 | using namespace LegalizeMutations; |
944 | return actionIf(Action: LegalizeAction::WidenScalar, |
945 | Predicate: all(P0: Predicate, P1: scalarOrEltNarrowerThan( |
946 | TypeIdx, Size: Ty.getScalarSizeInBits())), |
947 | Mutation: changeElementTo(TypeIdx: typeIdx(TypeIdx), Ty)); |
948 | } |
949 | |
950 | /// Ensure the vector size is at least as wide as VectorSize by promoting the |
951 | /// element. |
952 | LegalizeRuleSet &widenVectorEltsToVectorMinSize(unsigned TypeIdx, |
953 | unsigned VectorSize) { |
954 | using namespace LegalityPredicates; |
955 | using namespace LegalizeMutations; |
956 | return actionIf( |
957 | Action: LegalizeAction::WidenScalar, |
958 | Predicate: [=](const LegalityQuery &Query) { |
959 | const LLT VecTy = Query.Types[TypeIdx]; |
960 | return VecTy.isVector() && !VecTy.isScalable() && |
961 | VecTy.getSizeInBits() < VectorSize; |
962 | }, |
963 | Mutation: [=](const LegalityQuery &Query) { |
964 | const LLT VecTy = Query.Types[TypeIdx]; |
965 | unsigned NumElts = VecTy.getNumElements(); |
966 | unsigned MinSize = VectorSize / NumElts; |
967 | LLT NewTy = LLT::fixed_vector(NumElements: NumElts, ScalarTy: LLT::scalar(SizeInBits: MinSize)); |
968 | return std::make_pair(x: TypeIdx, y&: NewTy); |
969 | }); |
970 | } |
971 | |
972 | /// Ensure the scalar is at least as wide as Ty. |
973 | LegalizeRuleSet &minScalar(unsigned TypeIdx, const LLT Ty) { |
974 | using namespace LegalityPredicates; |
975 | using namespace LegalizeMutations; |
976 | return actionIf(Action: LegalizeAction::WidenScalar, |
977 | Predicate: scalarNarrowerThan(TypeIdx, Size: Ty.getSizeInBits()), |
978 | Mutation: changeTo(TypeIdx: typeIdx(TypeIdx), Ty)); |
979 | } |
980 | |
981 | /// Ensure the scalar is at least as wide as Ty if condition is met. |
982 | LegalizeRuleSet &minScalarIf(LegalityPredicate Predicate, unsigned TypeIdx, |
983 | const LLT Ty) { |
984 | using namespace LegalityPredicates; |
985 | using namespace LegalizeMutations; |
986 | return actionIf( |
987 | Action: LegalizeAction::WidenScalar, |
988 | Predicate: [=](const LegalityQuery &Query) { |
989 | const LLT QueryTy = Query.Types[TypeIdx]; |
990 | return QueryTy.isScalar() && |
991 | QueryTy.getSizeInBits() < Ty.getSizeInBits() && |
992 | Predicate(Query); |
993 | }, |
994 | Mutation: changeTo(TypeIdx: typeIdx(TypeIdx), Ty)); |
995 | } |
996 | |
997 | /// Ensure the scalar is at most as wide as Ty. |
998 | LegalizeRuleSet &maxScalarOrElt(unsigned TypeIdx, const LLT Ty) { |
999 | using namespace LegalityPredicates; |
1000 | using namespace LegalizeMutations; |
1001 | return actionIf(Action: LegalizeAction::NarrowScalar, |
1002 | Predicate: scalarOrEltWiderThan(TypeIdx, Size: Ty.getScalarSizeInBits()), |
1003 | Mutation: changeElementTo(TypeIdx: typeIdx(TypeIdx), Ty)); |
1004 | } |
1005 | |
1006 | /// Ensure the scalar is at most as wide as Ty. |
1007 | LegalizeRuleSet &maxScalar(unsigned TypeIdx, const LLT Ty) { |
1008 | using namespace LegalityPredicates; |
1009 | using namespace LegalizeMutations; |
1010 | return actionIf(Action: LegalizeAction::NarrowScalar, |
1011 | Predicate: scalarWiderThan(TypeIdx, Size: Ty.getSizeInBits()), |
1012 | Mutation: changeTo(TypeIdx: typeIdx(TypeIdx), Ty)); |
1013 | } |
1014 | |
1015 | /// Conditionally limit the maximum size of the scalar. |
1016 | /// For example, when the maximum size of one type depends on the size of |
1017 | /// another such as extracting N bits from an M bit container. |
1018 | LegalizeRuleSet &maxScalarIf(LegalityPredicate Predicate, unsigned TypeIdx, |
1019 | const LLT Ty) { |
1020 | using namespace LegalityPredicates; |
1021 | using namespace LegalizeMutations; |
1022 | return actionIf( |
1023 | Action: LegalizeAction::NarrowScalar, |
1024 | Predicate: [=](const LegalityQuery &Query) { |
1025 | const LLT QueryTy = Query.Types[TypeIdx]; |
1026 | return QueryTy.isScalar() && |
1027 | QueryTy.getSizeInBits() > Ty.getSizeInBits() && |
1028 | Predicate(Query); |
1029 | }, |
1030 | Mutation: changeElementTo(TypeIdx: typeIdx(TypeIdx), Ty)); |
1031 | } |
1032 | |
1033 | /// Limit the range of scalar sizes to MinTy and MaxTy. |
1034 | LegalizeRuleSet &clampScalar(unsigned TypeIdx, const LLT MinTy, |
1035 | const LLT MaxTy) { |
1036 | assert(MinTy.isScalar() && MaxTy.isScalar() && "Expected scalar types" ); |
1037 | return minScalar(TypeIdx, Ty: MinTy).maxScalar(TypeIdx, Ty: MaxTy); |
1038 | } |
1039 | |
1040 | /// Limit the range of scalar sizes to MinTy and MaxTy. |
1041 | LegalizeRuleSet &clampScalarOrElt(unsigned TypeIdx, const LLT MinTy, |
1042 | const LLT MaxTy) { |
1043 | return minScalarOrElt(TypeIdx, Ty: MinTy).maxScalarOrElt(TypeIdx, Ty: MaxTy); |
1044 | } |
1045 | |
1046 | /// Widen the scalar to match the size of another. |
1047 | LegalizeRuleSet &minScalarSameAs(unsigned TypeIdx, unsigned LargeTypeIdx) { |
1048 | typeIdx(TypeIdx); |
1049 | return widenScalarIf( |
1050 | Predicate: [=](const LegalityQuery &Query) { |
1051 | return Query.Types[LargeTypeIdx].getScalarSizeInBits() > |
1052 | Query.Types[TypeIdx].getSizeInBits(); |
1053 | }, |
1054 | Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx, FromTypeIdx: LargeTypeIdx)); |
1055 | } |
1056 | |
1057 | /// Narrow the scalar to match the size of another. |
1058 | LegalizeRuleSet &maxScalarSameAs(unsigned TypeIdx, unsigned NarrowTypeIdx) { |
1059 | typeIdx(TypeIdx); |
1060 | return narrowScalarIf( |
1061 | Predicate: [=](const LegalityQuery &Query) { |
1062 | return Query.Types[NarrowTypeIdx].getScalarSizeInBits() < |
1063 | Query.Types[TypeIdx].getSizeInBits(); |
1064 | }, |
1065 | Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx, FromTypeIdx: NarrowTypeIdx)); |
1066 | } |
1067 | |
1068 | /// Change the type \p TypeIdx to have the same scalar size as type \p |
1069 | /// SameSizeIdx. |
1070 | LegalizeRuleSet &scalarSameSizeAs(unsigned TypeIdx, unsigned SameSizeIdx) { |
1071 | return minScalarSameAs(TypeIdx, LargeTypeIdx: SameSizeIdx) |
1072 | .maxScalarSameAs(TypeIdx, NarrowTypeIdx: SameSizeIdx); |
1073 | } |
1074 | |
1075 | /// Conditionally widen the scalar or elt to match the size of another. |
1076 | LegalizeRuleSet &minScalarEltSameAsIf(LegalityPredicate Predicate, |
1077 | unsigned TypeIdx, unsigned LargeTypeIdx) { |
1078 | typeIdx(TypeIdx); |
1079 | return widenScalarIf( |
1080 | Predicate: [=](const LegalityQuery &Query) { |
1081 | return Query.Types[LargeTypeIdx].getScalarSizeInBits() > |
1082 | Query.Types[TypeIdx].getScalarSizeInBits() && |
1083 | Predicate(Query); |
1084 | }, |
1085 | Mutation: [=](const LegalityQuery &Query) { |
1086 | LLT T = Query.Types[LargeTypeIdx]; |
1087 | if (T.isPointerVector()) |
1088 | T = T.changeElementType(NewEltTy: LLT::scalar(SizeInBits: T.getScalarSizeInBits())); |
1089 | return std::make_pair(x: TypeIdx, y&: T); |
1090 | }); |
1091 | } |
1092 | |
1093 | /// Conditionally narrow the scalar or elt to match the size of another. |
1094 | LegalizeRuleSet &maxScalarEltSameAsIf(LegalityPredicate Predicate, |
1095 | unsigned TypeIdx, |
1096 | unsigned SmallTypeIdx) { |
1097 | typeIdx(TypeIdx); |
1098 | return narrowScalarIf( |
1099 | Predicate: [=](const LegalityQuery &Query) { |
1100 | return Query.Types[SmallTypeIdx].getScalarSizeInBits() < |
1101 | Query.Types[TypeIdx].getScalarSizeInBits() && |
1102 | Predicate(Query); |
1103 | }, |
1104 | Mutation: [=](const LegalityQuery &Query) { |
1105 | LLT T = Query.Types[SmallTypeIdx]; |
1106 | return std::make_pair(x: TypeIdx, y&: T); |
1107 | }); |
1108 | } |
1109 | |
1110 | /// Add more elements to the vector to reach the next power of two. |
1111 | /// No effect if the type is not a vector or the element count is a power of |
1112 | /// two. |
1113 | LegalizeRuleSet &moreElementsToNextPow2(unsigned TypeIdx) { |
1114 | using namespace LegalityPredicates; |
1115 | return actionIf(Action: LegalizeAction::MoreElements, |
1116 | Predicate: numElementsNotPow2(TypeIdx: typeIdx(TypeIdx)), |
1117 | Mutation: LegalizeMutations::moreElementsToNextPow2(TypeIdx)); |
1118 | } |
1119 | |
1120 | /// Limit the number of elements in EltTy vectors to at least MinElements. |
1121 | LegalizeRuleSet &clampMinNumElements(unsigned TypeIdx, const LLT EltTy, |
1122 | unsigned MinElements) { |
1123 | // Mark the type index as covered: |
1124 | typeIdx(TypeIdx); |
1125 | return actionIf( |
1126 | Action: LegalizeAction::MoreElements, |
1127 | Predicate: [=](const LegalityQuery &Query) { |
1128 | LLT VecTy = Query.Types[TypeIdx]; |
1129 | return VecTy.isVector() && VecTy.getElementType() == EltTy && |
1130 | VecTy.getNumElements() < MinElements; |
1131 | }, |
1132 | Mutation: [=](const LegalityQuery &Query) { |
1133 | LLT VecTy = Query.Types[TypeIdx]; |
1134 | return std::make_pair( |
1135 | x: TypeIdx, y: LLT::fixed_vector(NumElements: MinElements, ScalarTy: VecTy.getElementType())); |
1136 | }); |
1137 | } |
1138 | |
1139 | /// Set number of elements to nearest larger multiple of NumElts. |
1140 | LegalizeRuleSet &alignNumElementsTo(unsigned TypeIdx, const LLT EltTy, |
1141 | unsigned NumElts) { |
1142 | typeIdx(TypeIdx); |
1143 | return actionIf( |
1144 | Action: LegalizeAction::MoreElements, |
1145 | Predicate: [=](const LegalityQuery &Query) { |
1146 | LLT VecTy = Query.Types[TypeIdx]; |
1147 | return VecTy.isVector() && VecTy.getElementType() == EltTy && |
1148 | (VecTy.getNumElements() % NumElts != 0); |
1149 | }, |
1150 | Mutation: [=](const LegalityQuery &Query) { |
1151 | LLT VecTy = Query.Types[TypeIdx]; |
1152 | unsigned NewSize = alignTo(Value: VecTy.getNumElements(), Align: NumElts); |
1153 | return std::make_pair( |
1154 | x: TypeIdx, y: LLT::fixed_vector(NumElements: NewSize, ScalarTy: VecTy.getElementType())); |
1155 | }); |
1156 | } |
1157 | |
1158 | /// Limit the number of elements in EltTy vectors to at most MaxElements. |
1159 | LegalizeRuleSet &clampMaxNumElements(unsigned TypeIdx, const LLT EltTy, |
1160 | unsigned MaxElements) { |
1161 | // Mark the type index as covered: |
1162 | typeIdx(TypeIdx); |
1163 | return actionIf( |
1164 | Action: LegalizeAction::FewerElements, |
1165 | Predicate: [=](const LegalityQuery &Query) { |
1166 | LLT VecTy = Query.Types[TypeIdx]; |
1167 | return VecTy.isVector() && VecTy.getElementType() == EltTy && |
1168 | VecTy.getNumElements() > MaxElements; |
1169 | }, |
1170 | Mutation: [=](const LegalityQuery &Query) { |
1171 | LLT VecTy = Query.Types[TypeIdx]; |
1172 | LLT NewTy = LLT::scalarOrVector(EC: ElementCount::getFixed(MinVal: MaxElements), |
1173 | ScalarTy: VecTy.getElementType()); |
1174 | return std::make_pair(x: TypeIdx, y&: NewTy); |
1175 | }); |
1176 | } |
1177 | /// Limit the number of elements for the given vectors to at least MinTy's |
1178 | /// number of elements and at most MaxTy's number of elements. |
1179 | /// |
1180 | /// No effect if the type is not a vector or does not have the same element |
1181 | /// type as the constraints. |
1182 | /// The element type of MinTy and MaxTy must match. |
1183 | LegalizeRuleSet &clampNumElements(unsigned TypeIdx, const LLT MinTy, |
1184 | const LLT MaxTy) { |
1185 | assert(MinTy.getElementType() == MaxTy.getElementType() && |
1186 | "Expected element types to agree" ); |
1187 | |
1188 | const LLT EltTy = MinTy.getElementType(); |
1189 | return clampMinNumElements(TypeIdx, EltTy, MinElements: MinTy.getNumElements()) |
1190 | .clampMaxNumElements(TypeIdx, EltTy, MaxElements: MaxTy.getNumElements()); |
1191 | } |
1192 | |
1193 | /// Express \p EltTy vectors strictly using vectors with \p NumElts elements |
1194 | /// (or scalars when \p NumElts equals 1). |
1195 | /// First pad with undef elements to nearest larger multiple of \p NumElts. |
1196 | /// Then perform split with all sub-instructions having the same type. |
1197 | /// Using clampMaxNumElements (non-strict) can result in leftover instruction |
1198 | /// with different type (fewer elements then \p NumElts or scalar). |
1199 | /// No effect if the type is not a vector. |
1200 | LegalizeRuleSet &clampMaxNumElementsStrict(unsigned TypeIdx, const LLT EltTy, |
1201 | unsigned NumElts) { |
1202 | return alignNumElementsTo(TypeIdx, EltTy, NumElts) |
1203 | .clampMaxNumElements(TypeIdx, EltTy, MaxElements: NumElts); |
1204 | } |
1205 | |
1206 | /// Fallback on the previous implementation. This should only be used while |
1207 | /// porting a rule. |
1208 | LegalizeRuleSet &fallback() { |
1209 | add(Rule: {always, LegalizeAction::UseLegacyRules}); |
1210 | return *this; |
1211 | } |
1212 | |
1213 | /// Check if there is no type index which is obviously not handled by the |
1214 | /// LegalizeRuleSet in any way at all. |
1215 | /// \pre Type indices of the opcode form a dense [0, \p NumTypeIdxs) set. |
1216 | bool verifyTypeIdxsCoverage(unsigned NumTypeIdxs) const; |
1217 | /// Check if there is no imm index which is obviously not handled by the |
1218 | /// LegalizeRuleSet in any way at all. |
1219 | /// \pre Type indices of the opcode form a dense [0, \p NumTypeIdxs) set. |
1220 | bool verifyImmIdxsCoverage(unsigned NumImmIdxs) const; |
1221 | |
1222 | /// Apply the ruleset to the given LegalityQuery. |
1223 | LegalizeActionStep apply(const LegalityQuery &Query) const; |
1224 | }; |
1225 | |
1226 | class LegalizerInfo { |
1227 | public: |
1228 | virtual ~LegalizerInfo() = default; |
1229 | |
1230 | const LegacyLegalizerInfo &getLegacyLegalizerInfo() const { |
1231 | return LegacyInfo; |
1232 | } |
1233 | LegacyLegalizerInfo &getLegacyLegalizerInfo() { return LegacyInfo; } |
1234 | |
1235 | unsigned getOpcodeIdxForOpcode(unsigned Opcode) const; |
1236 | unsigned getActionDefinitionsIdx(unsigned Opcode) const; |
1237 | |
1238 | /// Perform simple self-diagnostic and assert if there is anything obviously |
1239 | /// wrong with the actions set up. |
1240 | void verify(const MCInstrInfo &MII) const; |
1241 | |
1242 | /// Get the action definitions for the given opcode. Use this to run a |
1243 | /// LegalityQuery through the definitions. |
1244 | const LegalizeRuleSet &getActionDefinitions(unsigned Opcode) const; |
1245 | |
1246 | /// Get the action definition builder for the given opcode. Use this to define |
1247 | /// the action definitions. |
1248 | /// |
1249 | /// It is an error to request an opcode that has already been requested by the |
1250 | /// multiple-opcode variant. |
1251 | LegalizeRuleSet &getActionDefinitionsBuilder(unsigned Opcode); |
1252 | |
1253 | /// Get the action definition builder for the given set of opcodes. Use this |
1254 | /// to define the action definitions for multiple opcodes at once. The first |
1255 | /// opcode given will be considered the representative opcode and will hold |
1256 | /// the definitions whereas the other opcodes will be configured to refer to |
1257 | /// the representative opcode. This lowers memory requirements and very |
1258 | /// slightly improves performance. |
1259 | /// |
1260 | /// It would be very easy to introduce unexpected side-effects as a result of |
1261 | /// this aliasing if it were permitted to request different but intersecting |
1262 | /// sets of opcodes but that is difficult to keep track of. It is therefore an |
1263 | /// error to request the same opcode twice using this API, to request an |
1264 | /// opcode that already has definitions, or to use the single-opcode API on an |
1265 | /// opcode that has already been requested by this API. |
1266 | LegalizeRuleSet & |
1267 | getActionDefinitionsBuilder(std::initializer_list<unsigned> Opcodes); |
1268 | void aliasActionDefinitions(unsigned OpcodeTo, unsigned OpcodeFrom); |
1269 | |
1270 | /// Determine what action should be taken to legalize the described |
1271 | /// instruction. Requires computeTables to have been called. |
1272 | /// |
1273 | /// \returns a description of the next legalization step to perform. |
1274 | LegalizeActionStep getAction(const LegalityQuery &Query) const; |
1275 | |
1276 | /// Determine what action should be taken to legalize the given generic |
1277 | /// instruction. |
1278 | /// |
1279 | /// \returns a description of the next legalization step to perform. |
1280 | LegalizeActionStep getAction(const MachineInstr &MI, |
1281 | const MachineRegisterInfo &MRI) const; |
1282 | |
1283 | bool isLegal(const LegalityQuery &Query) const { |
1284 | return getAction(Query).Action == LegalizeAction::Legal; |
1285 | } |
1286 | |
1287 | bool isLegalOrCustom(const LegalityQuery &Query) const { |
1288 | auto Action = getAction(Query).Action; |
1289 | return Action == LegalizeAction::Legal || Action == LegalizeAction::Custom; |
1290 | } |
1291 | |
1292 | bool isLegal(const MachineInstr &MI, const MachineRegisterInfo &MRI) const; |
1293 | bool isLegalOrCustom(const MachineInstr &MI, |
1294 | const MachineRegisterInfo &MRI) const; |
1295 | |
1296 | /// Called for instructions with the Custom LegalizationAction. |
1297 | virtual bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI, |
1298 | LostDebugLocObserver &LocObserver) const { |
1299 | llvm_unreachable("must implement this if custom action is used" ); |
1300 | } |
1301 | |
1302 | /// \returns true if MI is either legal or has been legalized and false if not |
1303 | /// legal. |
1304 | /// Return true if MI is either legal or has been legalized and false |
1305 | /// if not legal. |
1306 | virtual bool legalizeIntrinsic(LegalizerHelper &Helper, |
1307 | MachineInstr &MI) const { |
1308 | return true; |
1309 | } |
1310 | |
1311 | /// Return the opcode (SEXT/ZEXT/ANYEXT) that should be performed while |
1312 | /// widening a constant of type SmallTy which targets can override. |
1313 | /// For eg, the DAG does (SmallTy.isByteSized() ? G_SEXT : G_ZEXT) which |
1314 | /// will be the default. |
1315 | virtual unsigned getExtOpcodeForWideningConstant(LLT SmallTy) const; |
1316 | |
1317 | private: |
1318 | static const int FirstOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_START; |
1319 | static const int LastOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_END; |
1320 | |
1321 | LegalizeRuleSet RulesForOpcode[LastOp - FirstOp + 1]; |
1322 | LegacyLegalizerInfo LegacyInfo; |
1323 | }; |
1324 | |
1325 | #ifndef NDEBUG |
1326 | /// Checks that MIR is fully legal, returns an illegal instruction if it's not, |
1327 | /// nullptr otherwise |
1328 | const MachineInstr *machineFunctionIsIllegal(const MachineFunction &MF); |
1329 | #endif |
1330 | |
1331 | } // end namespace llvm. |
1332 | |
1333 | #endif // LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H |
1334 | |