1//===--- arm_sve.td - ARM SVE compiler interface ------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the TableGen definitions from which the ARM SVE header
10// file will be generated. See:
11//
12// https://developer.arm.com/architectures/system-architectures/software-standards/acle
13//
14//===----------------------------------------------------------------------===//
15
16//===----------------------------------------------------------------------===//
17// Instruction definitions
18//===----------------------------------------------------------------------===//
19// Every intrinsic subclasses "Inst". An intrinsic has a name, a prototype and
20// a sequence of typespecs.
21//
22// The name is the base name of the intrinsic, for example "svld1". This is
23// then mangled by the tblgen backend to add type information ("svld1_s16").
24//
25// A typespec is a sequence of uppercase characters (modifiers) followed by one
26// lowercase character. A typespec encodes a particular "base type" of the
27// intrinsic.
28//
29// An example typespec is "Us" - unsigned short - svuint16_t. The available
30// typespec codes are given below.
31//
32// The string given to an Inst class is a sequence of typespecs. The intrinsic
33// is instantiated for every typespec in the sequence. For example "sdUsUd".
34//
35// The prototype is a string that defines the return type of the intrinsic
36// and the type of each argument. The return type and every argument gets a
37// "modifier" that can change in some way the "base type" of the intrinsic.
38//
39// The modifier 'd' means "default" and does not modify the base type in any
40// way. The available modifiers are given below.
41//
42// Typespecs
43// ---------
44// c: char
45// s: short
46// i: int
47// l: long
48// f: float
49// h: half-float
50// d: double
51// b: bfloat
52
53// Typespec modifiers
54// ------------------
55// P: boolean
56// U: unsigned
57
58// Prototype modifiers
59// -------------------
60// prototype: return (arg, arg, ...)
61//
62// 2,3,4: array of default vectors
63// v: void
64// x: vector of signed integers
65// u: vector of unsigned integers
66// d: default
67// c: const pointer type
68// P: predicate type
69// s: scalar of element type
70// a: scalar of element type (splat to vector type)
71// R: scalar of 1/2 width element type (splat to vector type)
72// r: scalar of 1/4 width element type (splat to vector type)
73// @: unsigned scalar of 1/4 width element type (splat to vector type)
74// e: 1/2 width unsigned elements, 2x element count
75// b: 1/4 width unsigned elements, 4x element count
76// h: 1/2 width elements, 2x element count
77// q: 1/4 width elements, 4x element count
78// o: 4x width elements, 1/4 element count
79//
80// w: vector of element type promoted to 64bits, vector maintains
81// signedness of its element type.
82// f: element type promoted to uint64_t (splat to vector type)
83// j: element type promoted to 64bits (splat to vector type)
84// K: element type bitcast to a signed integer (splat to vector type)
85// L: element type bitcast to an unsigned integer (splat to vector type)
86//
87// i: constant uint64_t
88// k: int32_t
89// l: int64_t
90// m: uint32_t
91// n: uint64_t
92
93// t: svint32_t
94// z: svuint32_t
95// g: svuint64_t
96// O: svfloat16_t
97// M: svfloat32_t
98// N: svfloat64_t
99
100// J: Prefetch type (sv_prfop)
101// A: pointer to int8_t
102// B: pointer to int16_t
103// C: pointer to int32_t
104// D: pointer to int64_t
105
106// E: pointer to uint8_t
107// F: pointer to uint16_t
108// G: pointer to uint32_t
109// H: pointer to uint64_t
110
111// Q: const pointer to void
112
113// S: const pointer to int8_t
114// T: const pointer to int16_t
115// U: const pointer to int32_t
116// V: const pointer to int64_t
117//
118// W: const pointer to uint8_t
119// X: const pointer to uint16_t
120// Y: const pointer to uint32_t
121// Z: const pointer to uint64_t
122
123class MergeType<int val, string suffix=""> {
124 int Value = val;
125 string Suffix = suffix;
126}
127def MergeNone : MergeType<0>;
128def MergeAny : MergeType<1, "_x">;
129def MergeOp1 : MergeType<2, "_m">;
130def MergeZero : MergeType<3, "_z">;
131def MergeAnyExp : MergeType<4, "_x">; // Use merged builtin with explicit
132def MergeZeroExp : MergeType<5, "_z">; // generation of its inactive argument.
133
134class EltType<int val> {
135 int Value = val;
136}
137def EltTyInvalid : EltType<0>;
138def EltTyInt8 : EltType<1>;
139def EltTyInt16 : EltType<2>;
140def EltTyInt32 : EltType<3>;
141def EltTyInt64 : EltType<4>;
142def EltTyFloat16 : EltType<5>;
143def EltTyFloat32 : EltType<6>;
144def EltTyFloat64 : EltType<7>;
145def EltTyBool8 : EltType<8>;
146def EltTyBool16 : EltType<9>;
147def EltTyBool32 : EltType<10>;
148def EltTyBool64 : EltType<11>;
149def EltTyBFloat16 : EltType<12>;
150
151class MemEltType<int val> {
152 int Value = val;
153}
154def MemEltTyDefault : MemEltType<0>;
155def MemEltTyInt8 : MemEltType<1>;
156def MemEltTyInt16 : MemEltType<2>;
157def MemEltTyInt32 : MemEltType<3>;
158def MemEltTyInt64 : MemEltType<4>;
159
160class FlagType<int val> {
161 int Value = val;
162}
163
164// These must be kept in sync with the flags in utils/TableGen/SveEmitter.h
165// and include/clang/Basic/TargetBuiltins.h
166def NoFlags : FlagType<0x00000000>;
167def FirstEltType : FlagType<0x00000001>;
168// : :
169// : :
170def EltTypeMask : FlagType<0x0000000f>;
171def FirstMemEltType : FlagType<0x00000010>;
172// : :
173// : :
174def MemEltTypeMask : FlagType<0x00000070>;
175def FirstMergeTypeMask : FlagType<0x00000080>;
176// : :
177// : :
178def MergeTypeMask : FlagType<0x00000380>;
179def FirstSplatOperand : FlagType<0x00000400>;
180// : :
181// These flags are used to specify which scalar operand
182// needs to be duplicated/splatted into a vector.
183// : :
184def SplatOperandMask : FlagType<0x00001C00>;
185def IsLoad : FlagType<0x00002000>;
186def IsStore : FlagType<0x00004000>;
187def IsGatherLoad : FlagType<0x00008000>;
188def IsScatterStore : FlagType<0x00010000>;
189def IsStructLoad : FlagType<0x00020000>;
190def IsStructStore : FlagType<0x00040000>;
191def IsZExtReturn : FlagType<0x00080000>; // Return value is sign-extend by default
192def IsOverloadNone : FlagType<0x00100000>; // Intrinsic does not take any overloaded types.
193def IsOverloadWhile : FlagType<0x00200000>; // Use {default type, typeof(operand1)} as overloaded types.
194def IsOverloadWhileRW : FlagType<0x00400000>; // Use {pred(default type), typeof(operand0)} as overloaded types.
195def IsOverloadCvt : FlagType<0x00800000>; // Use {typeof(operand0), typeof(last operand)} as overloaded types.
196def OverloadKindMask : FlagType<0x00E00000>; // When the masked values are all '0', the default type is used as overload type.
197def IsByteIndexed : FlagType<0x01000000>;
198def IsAppendSVALL : FlagType<0x02000000>; // Appends SV_ALL as the last operand.
199def IsInsertOp1SVALL : FlagType<0x04000000>; // Inserts SV_ALL as the second operand.
200def IsPrefetch : FlagType<0x08000000>; // Contiguous prefetches.
201def IsGatherPrefetch : FlagType<0x10000000>;
202def ReverseCompare : FlagType<0x20000000>; // Compare operands must be swapped.
203def ReverseUSDOT : FlagType<0x40000000>; // Unsigned/signed operands must be swapped.
204def IsUndef : FlagType<0x80000000>; // Codegen `undef` of given type.
205def IsTupleCreate : FlagType<0x100000000>;
206def IsTupleGet : FlagType<0x200000000>;
207def IsTupleSet : FlagType<0x400000000>;
208
209// These must be kept in sync with the flags in include/clang/Basic/TargetBuiltins.h
210class ImmCheckType<int val> {
211 int Value = val;
212}
213def ImmCheck0_31 : ImmCheckType<0>; // 0..31 (used for e.g. predicate patterns)
214def ImmCheck1_16 : ImmCheckType<1>; // 1..16
215def ImmCheckExtract : ImmCheckType<2>; // 0..(2048/sizeinbits(elt) - 1)
216def ImmCheckShiftRight : ImmCheckType<3>; // 1..sizeinbits(elt)
217def ImmCheckShiftRightNarrow : ImmCheckType<4>; // 1..sizeinbits(elt)/2
218def ImmCheckShiftLeft : ImmCheckType<5>; // 0..(sizeinbits(elt) - 1)
219def ImmCheck0_7 : ImmCheckType<6>; // 0..7
220def ImmCheckLaneIndex : ImmCheckType<7>; // 0..(128/(1*sizeinbits(elt)) - 1)
221def ImmCheckLaneIndexCompRotate : ImmCheckType<8>; // 0..(128/(2*sizeinbits(elt)) - 1)
222def ImmCheckLaneIndexDot : ImmCheckType<9>; // 0..(128/(4*sizeinbits(elt)) - 1)
223def ImmCheckComplexRot90_270 : ImmCheckType<10>; // [90,270]
224def ImmCheckComplexRotAll90 : ImmCheckType<11>; // [0, 90, 180,270]
225def ImmCheck0_13 : ImmCheckType<12>; // 0..13
226def ImmCheck0_1 : ImmCheckType<13>; // 0..1
227def ImmCheck0_2 : ImmCheckType<14>; // 0..2
228def ImmCheck0_3 : ImmCheckType<15>; // 0..3
229
230class ImmCheck<int arg, ImmCheckType kind, int eltSizeArg = -1> {
231 int Arg = arg;
232 int EltSizeArg = eltSizeArg;
233 ImmCheckType Kind = kind;
234}
235
236class Inst<string n, string p, string t, MergeType mt, string i,
237 list<FlagType> ft, list<ImmCheck> ch, MemEltType met> {
238 string Name = n;
239 string Prototype = p;
240 string Types = t;
241 string ArchGuard = "";
242 int Merge = mt.Value;
243 string MergeSuffix = mt.Suffix;
244 string LLVMIntrinsic = i;
245 list<FlagType> Flags = ft;
246 list<ImmCheck> ImmChecks = ch;
247 int MemEltType = met.Value;
248}
249
250// SInst: Instruction with signed/unsigned suffix (e.g., "s8", "u8")
251class SInst<string n, string p, string t, MergeType mt, string i = "",
252 list<FlagType> ft = [], list<ImmCheck> ch = []>
253 : Inst<n, p, t, mt, i, ft, ch, MemEltTyDefault> {
254}
255
256// MInst: Instructions which access memory
257class MInst<string n, string p, string t, list<FlagType> f,
258 MemEltType met = MemEltTyDefault, string i = "">
259 : Inst<n, p, t, MergeNone, i, f, [], met> {
260}
261
262////////////////////////////////////////////////////////////////////////////////
263// Loads
264
265// Load one vector (scalar base)
266def SVLD1 : MInst<"svld1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad], MemEltTyDefault, "aarch64_sve_ld1">;
267def SVLD1SB : MInst<"svld1sb_{d}", "dPS", "silUsUiUl", [IsLoad], MemEltTyInt8, "aarch64_sve_ld1">;
268def SVLD1UB : MInst<"svld1ub_{d}", "dPW", "silUsUiUl", [IsLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ld1">;
269def SVLD1SH : MInst<"svld1sh_{d}", "dPT", "ilUiUl", [IsLoad], MemEltTyInt16, "aarch64_sve_ld1">;
270def SVLD1UH : MInst<"svld1uh_{d}", "dPX", "ilUiUl", [IsLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1">;
271def SVLD1SW : MInst<"svld1sw_{d}", "dPU", "lUl", [IsLoad], MemEltTyInt32, "aarch64_sve_ld1">;
272def SVLD1UW : MInst<"svld1uw_{d}", "dPY", "lUl", [IsLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1">;
273
274let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
275 def SVLD1_BF : MInst<"svld1[_{2}]", "dPc", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ld1">;
276 def SVLD1_VNUM_BF : MInst<"svld1_vnum[_{2}]", "dPcl", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ld1">;
277}
278
279// Load one vector (scalar base, VL displacement)
280def SVLD1_VNUM : MInst<"svld1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfd", [IsLoad], MemEltTyDefault, "aarch64_sve_ld1">;
281def SVLD1SB_VNUM : MInst<"svld1sb_vnum_{d}", "dPSl", "silUsUiUl", [IsLoad], MemEltTyInt8, "aarch64_sve_ld1">;
282def SVLD1UB_VNUM : MInst<"svld1ub_vnum_{d}", "dPWl", "silUsUiUl", [IsLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ld1">;
283def SVLD1SH_VNUM : MInst<"svld1sh_vnum_{d}", "dPTl", "ilUiUl", [IsLoad], MemEltTyInt16, "aarch64_sve_ld1">;
284def SVLD1UH_VNUM : MInst<"svld1uh_vnum_{d}", "dPXl", "ilUiUl", [IsLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1">;
285def SVLD1SW_VNUM : MInst<"svld1sw_vnum_{d}", "dPUl", "lUl", [IsLoad], MemEltTyInt32, "aarch64_sve_ld1">;
286def SVLD1UW_VNUM : MInst<"svld1uw_vnum_{d}", "dPYl", "lUl", [IsLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1">;
287
288// Load one vector (vector base)
289def SVLD1_GATHER_BASES_U : MInst<"svld1_gather[_{2}base]_{d}", "dPu", "ilUiUlfd", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ld1_gather_scalar_offset">;
290def SVLD1SB_GATHER_BASES_U : MInst<"svld1sb_gather[_{2}base]_{d}", "dPu", "ilUiUl", [IsGatherLoad], MemEltTyInt8, "aarch64_sve_ld1_gather_scalar_offset">;
291def SVLD1UB_GATHER_BASES_U : MInst<"svld1ub_gather[_{2}base]_{d}", "dPu", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ld1_gather_scalar_offset">;
292def SVLD1SH_GATHER_BASES_U : MInst<"svld1sh_gather[_{2}base]_{d}", "dPu", "ilUiUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ld1_gather_scalar_offset">;
293def SVLD1UH_GATHER_BASES_U : MInst<"svld1uh_gather[_{2}base]_{d}", "dPu", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_scalar_offset">;
294def SVLD1SW_GATHER_BASES_U : MInst<"svld1sw_gather[_{2}base]_{d}", "dPu", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ld1_gather_scalar_offset">;
295def SVLD1UW_GATHER_BASES_U : MInst<"svld1uw_gather[_{2}base]_{d}", "dPu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1_gather_scalar_offset">;
296
297// Load one vector (scalar base, signed vector offset in bytes)
298def SVLD1_GATHER_64B_OFFSETS_S : MInst<"svld1_gather_[{3}]offset[_{d}]", "dPcx", "lUld", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ld1_gather">;
299def SVLD1SB_GATHER_64B_OFFSETS_S : MInst<"svld1sb_gather_[{3}]offset_{d}", "dPSx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ld1_gather">;
300def SVLD1UB_GATHER_64B_OFFSETS_S : MInst<"svld1ub_gather_[{3}]offset_{d}", "dPWx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ld1_gather">;
301def SVLD1SH_GATHER_64B_OFFSETS_S : MInst<"svld1sh_gather_[{3}]offset_{d}", "dPTx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ld1_gather">;
302def SVLD1UH_GATHER_64B_OFFSETS_S : MInst<"svld1uh_gather_[{3}]offset_{d}", "dPXx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather">;
303def SVLD1SW_GATHER_64B_OFFSETS_S : MInst<"svld1sw_gather_[{3}]offset_{d}", "dPUx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ld1_gather">;
304def SVLD1UW_GATHER_64B_OFFSETS_S : MInst<"svld1uw_gather_[{3}]offset_{d}", "dPYx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1_gather">;
305
306def SVLD1_GATHER_32B_OFFSETS_S : MInst<"svld1_gather_[{3}]offset[_{d}]", "dPcx", "iUif", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ld1_gather_sxtw">;
307def SVLD1SB_GATHER_32B_OFFSETS_S : MInst<"svld1sb_gather_[{3}]offset_{d}", "dPSx", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ld1_gather_sxtw">;
308def SVLD1UB_GATHER_32B_OFFSETS_S : MInst<"svld1ub_gather_[{3}]offset_{d}", "dPWx", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ld1_gather_sxtw">;
309def SVLD1SH_GATHER_32B_OFFSETS_S : MInst<"svld1sh_gather_[{3}]offset_{d}", "dPTx", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ld1_gather_sxtw">;
310def SVLD1UH_GATHER_32B_OFFSETS_S : MInst<"svld1uh_gather_[{3}]offset_{d}", "dPXx", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_sxtw">;
311
312// Load one vector (scalar base, unsigned vector offset in bytes)
313def SVLD1_GATHER_64B_OFFSETS_U : MInst<"svld1_gather_[{3}]offset[_{d}]", "dPcu", "lUld", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ld1_gather">;
314def SVLD1SB_GATHER_64B_OFFSETS_U : MInst<"svld1sb_gather_[{3}]offset_{d}", "dPSu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ld1_gather">;
315def SVLD1UB_GATHER_64B_OFFSETS_U : MInst<"svld1ub_gather_[{3}]offset_{d}", "dPWu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ld1_gather">;
316def SVLD1SH_GATHER_64B_OFFSETS_U : MInst<"svld1sh_gather_[{3}]offset_{d}", "dPTu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ld1_gather">;
317def SVLD1UH_GATHER_64B_OFFSETS_U : MInst<"svld1uh_gather_[{3}]offset_{d}", "dPXu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather">;
318def SVLD1SW_GATHER_64B_OFFSETS_U : MInst<"svld1sw_gather_[{3}]offset_{d}", "dPUu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ld1_gather">;
319def SVLD1UW_GATHER_64B_OFFSETS_U : MInst<"svld1uw_gather_[{3}]offset_{d}", "dPYu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1_gather">;
320
321def SVLD1_GATHER_32B_OFFSETS_U : MInst<"svld1_gather_[{3}]offset[_{d}]", "dPcu", "iUif", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ld1_gather_uxtw">;
322def SVLD1SB_GATHER_32B_OFFSETS_U : MInst<"svld1sb_gather_[{3}]offset_{d}", "dPSu", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ld1_gather_uxtw">;
323def SVLD1UB_GATHER_32B_OFFSETS_U : MInst<"svld1ub_gather_[{3}]offset_{d}", "dPWu", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ld1_gather_uxtw">;
324def SVLD1SH_GATHER_32B_OFFSETS_U : MInst<"svld1sh_gather_[{3}]offset_{d}", "dPTu", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ld1_gather_uxtw">;
325def SVLD1UH_GATHER_32B_OFFSETS_U : MInst<"svld1uh_gather_[{3}]offset_{d}", "dPXu", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_uxtw">;
326
327// Load one vector (vector base, signed scalar offset in bytes)
328def SVLD1_GATHER_OFFSET_S : MInst<"svld1_gather[_{2}base]_offset_{d}", "dPul", "ilUiUlfd", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ld1_gather_scalar_offset">;
329def SVLD1SB_GATHER_OFFSET_S : MInst<"svld1sb_gather[_{2}base]_offset_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ld1_gather_scalar_offset">;
330def SVLD1UB_GATHER_OFFSET_S : MInst<"svld1ub_gather[_{2}base]_offset_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ld1_gather_scalar_offset">;
331def SVLD1SH_GATHER_OFFSET_S : MInst<"svld1sh_gather[_{2}base]_offset_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ld1_gather_scalar_offset">;
332def SVLD1UH_GATHER_OFFSET_S : MInst<"svld1uh_gather[_{2}base]_offset_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_scalar_offset">;
333def SVLD1SW_GATHER_OFFSET_S : MInst<"svld1sw_gather[_{2}base]_offset_{d}", "dPul", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ld1_gather_scalar_offset">;
334def SVLD1UW_GATHER_OFFSET_S : MInst<"svld1uw_gather[_{2}base]_offset_{d}", "dPul", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1_gather_scalar_offset">;
335
336// Load one vector (scalar base, signed vector index)
337def SVLD1_GATHER_64B_INDICES_S : MInst<"svld1_gather_[{3}]index[_{d}]", "dPcx", "lUld", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ld1_gather_index">;
338def SVLD1SH_GATHER_64B_INDICES_S : MInst<"svld1sh_gather_[{3}]index_{d}", "dPTx", "lUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ld1_gather_index">;
339def SVLD1UH_GATHER_64B_INDICES_S : MInst<"svld1uh_gather_[{3}]index_{d}", "dPXx", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_index">;
340def SVLD1SW_GATHER_64B_INDICES_S : MInst<"svld1sw_gather_[{3}]index_{d}", "dPUx", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ld1_gather_index">;
341def SVLD1UW_GATHER_64B_INDICES_S : MInst<"svld1uw_gather_[{3}]index_{d}", "dPYx", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1_gather_index">;
342
343def SVLD1_GATHER_32B_INDICES_S : MInst<"svld1_gather_[{3}]index[_{d}]", "dPcx", "iUif", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ld1_gather_sxtw_index">;
344def SVLD1SH_GATHER_32B_INDICES_S : MInst<"svld1sh_gather_[{3}]index_{d}", "dPTx", "iUi", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ld1_gather_sxtw_index">;
345def SVLD1UH_GATHER_32B_INDICES_S : MInst<"svld1uh_gather_[{3}]index_{d}", "dPXx", "iUi", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_sxtw_index">;
346
347// Load one vector (scalar base, unsigned vector index)
348def SVLD1_GATHER_64B_INDICES_U : MInst<"svld1_gather_[{3}]index[_{d}]", "dPcu", "lUld", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ld1_gather_index">;
349def SVLD1SH_GATHER_64B_INDICES_U : MInst<"svld1sh_gather_[{3}]index_{d}", "dPTu", "lUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ld1_gather_index">;
350def SVLD1UH_GATHER_64B_INDICES_U : MInst<"svld1uh_gather_[{3}]index_{d}", "dPXu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_index">;
351def SVLD1SW_GATHER_64B_INDICES_U : MInst<"svld1sw_gather_[{3}]index_{d}", "dPUu", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ld1_gather_index">;
352def SVLD1UW_GATHER_64B_INDICES_U : MInst<"svld1uw_gather_[{3}]index_{d}", "dPYu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1_gather_index">;
353
354def SVLD1_GATHER_32B_INDICES_U : MInst<"svld1_gather_[{3}]index[_{d}]", "dPcu", "iUif", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ld1_gather_uxtw_index">;
355def SVLD1SH_GATHER_32B_INDICES_U : MInst<"svld1sh_gather_[{3}]index_{d}", "dPTu", "iUi", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ld1_gather_uxtw_index">;
356def SVLD1UH_GATHER_32B_INDICES_U : MInst<"svld1uh_gather_[{3}]index_{d}", "dPXu", "iUi", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_uxtw_index">;
357
358// Load one vector (vector base, signed scalar index)
359def SVLD1_GATHER_INDEX_S : MInst<"svld1_gather[_{2}base]_index_{d}", "dPul", "ilUiUlfd", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ld1_gather_scalar_offset">;
360def SVLD1SH_GATHER_INDEX_S : MInst<"svld1sh_gather[_{2}base]_index_{d}", "dPul", "ilUiUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ld1_gather_scalar_offset">;
361def SVLD1UH_GATHER_INDEX_S : MInst<"svld1uh_gather[_{2}base]_index_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_scalar_offset">;
362def SVLD1SW_GATHER_INDEX_S : MInst<"svld1sw_gather[_{2}base]_index_{d}", "dPul", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ld1_gather_scalar_offset">;
363def SVLD1UW_GATHER_INDEX_S : MInst<"svld1uw_gather[_{2}base]_index_{d}", "dPul", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1_gather_scalar_offset">;
364
365
366// First-faulting load one vector (scalar base)
367def SVLDFF1 : MInst<"svldff1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad], MemEltTyDefault, "aarch64_sve_ldff1">;
368def SVLDFF1SB : MInst<"svldff1sb_{d}", "dPS", "silUsUiUl", [IsLoad], MemEltTyInt8, "aarch64_sve_ldff1">;
369def SVLDFF1UB : MInst<"svldff1ub_{d}", "dPW", "silUsUiUl", [IsLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldff1">;
370def SVLDFF1SH : MInst<"svldff1sh_{d}", "dPT", "ilUiUl", [IsLoad], MemEltTyInt16, "aarch64_sve_ldff1">;
371def SVLDFF1UH : MInst<"svldff1uh_{d}", "dPX", "ilUiUl", [IsLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1">;
372def SVLDFF1SW : MInst<"svldff1sw_{d}", "dPU", "lUl", [IsLoad], MemEltTyInt32, "aarch64_sve_ldff1">;
373def SVLDFF1UW : MInst<"svldff1uw_{d}", "dPY", "lUl", [IsLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1">;
374
375// First-faulting load one vector (scalar base, VL displacement)
376def SVLDFF1_VNUM : MInst<"svldff1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfd", [IsLoad], MemEltTyDefault, "aarch64_sve_ldff1">;
377def SVLDFF1SB_VNUM : MInst<"svldff1sb_vnum_{d}", "dPSl", "silUsUiUl", [IsLoad], MemEltTyInt8, "aarch64_sve_ldff1">;
378def SVLDFF1UB_VNUM : MInst<"svldff1ub_vnum_{d}", "dPWl", "silUsUiUl", [IsLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldff1">;
379def SVLDFF1SH_VNUM : MInst<"svldff1sh_vnum_{d}", "dPTl", "ilUiUl", [IsLoad], MemEltTyInt16, "aarch64_sve_ldff1">;
380def SVLDFF1UH_VNUM : MInst<"svldff1uh_vnum_{d}", "dPXl", "ilUiUl", [IsLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1">;
381def SVLDFF1SW_VNUM : MInst<"svldff1sw_vnum_{d}", "dPUl", "lUl", [IsLoad], MemEltTyInt32, "aarch64_sve_ldff1">;
382def SVLDFF1UW_VNUM : MInst<"svldff1uw_vnum_{d}", "dPYl", "lUl", [IsLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1">;
383
384let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
385 def SVLDFF1_BF : MInst<"svldff1[_{2}]", "dPc", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ldff1">;
386 def SVLDFF1_VNUM_BF : MInst<"svldff1_vnum[_{2}]", "dPcl", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ldff1">;
387}
388
389// First-faulting load one vector (vector base)
390def SVLDFF1_GATHER_BASES_U : MInst<"svldff1_gather[_{2}base]_{d}", "dPu", "ilUiUlfd", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldff1_gather_scalar_offset">;
391def SVLDFF1SB_GATHER_BASES_U : MInst<"svldff1sb_gather[_{2}base]_{d}", "dPu", "ilUiUl", [IsGatherLoad], MemEltTyInt8, "aarch64_sve_ldff1_gather_scalar_offset">;
392def SVLDFF1UB_GATHER_BASES_U : MInst<"svldff1ub_gather[_{2}base]_{d}", "dPu", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldff1_gather_scalar_offset">;
393def SVLDFF1SH_GATHER_BASES_U : MInst<"svldff1sh_gather[_{2}base]_{d}", "dPu", "ilUiUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldff1_gather_scalar_offset">;
394def SVLDFF1UH_GATHER_BASES_U : MInst<"svldff1uh_gather[_{2}base]_{d}", "dPu", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_scalar_offset">;
395def SVLDFF1SW_GATHER_BASES_U : MInst<"svldff1sw_gather[_{2}base]_{d}", "dPu", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ldff1_gather_scalar_offset">;
396def SVLDFF1UW_GATHER_BASES_U : MInst<"svldff1uw_gather[_{2}base]_{d}", "dPu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1_gather_scalar_offset">;
397
398// First-faulting load one vector (scalar base, signed vector offset in bytes)
399def SVLDFF1_GATHER_64B_OFFSETS_S : MInst<"svldff1_gather_[{3}]offset[_{d}]", "dPcx", "lUld", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldff1_gather">;
400def SVLDFF1SB_GATHER_64B_OFFSETS_S : MInst<"svldff1sb_gather_[{3}]offset_{d}", "dPSx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldff1_gather">;
401def SVLDFF1UB_GATHER_64B_OFFSETS_S : MInst<"svldff1ub_gather_[{3}]offset_{d}", "dPWx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldff1_gather">;
402def SVLDFF1SH_GATHER_64B_OFFSETS_S : MInst<"svldff1sh_gather_[{3}]offset_{d}", "dPTx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldff1_gather">;
403def SVLDFF1UH_GATHER_64B_OFFSETS_S : MInst<"svldff1uh_gather_[{3}]offset_{d}", "dPXx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather">;
404def SVLDFF1SW_GATHER_64B_OFFSETS_S : MInst<"svldff1sw_gather_[{3}]offset_{d}", "dPUx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ldff1_gather">;
405def SVLDFF1UW_GATHER_64B_OFFSETS_S : MInst<"svldff1uw_gather_[{3}]offset_{d}", "dPYx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1_gather">;
406
407def SVLDFF1_GATHER_32B_OFFSETS_S : MInst<"svldff1_gather_[{3}]offset[_{d}]", "dPcx", "iUif", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldff1_gather_sxtw">;
408def SVLDFF1SB_GATHER_32B_OFFSETS_S : MInst<"svldff1sb_gather_[{3}]offset_{d}", "dPSx", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldff1_gather_sxtw">;
409def SVLDFF1UB_GATHER_32B_OFFSETS_S : MInst<"svldff1ub_gather_[{3}]offset_{d}", "dPWx", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldff1_gather_sxtw">;
410def SVLDFF1SH_GATHER_32B_OFFSETS_S : MInst<"svldff1sh_gather_[{3}]offset_{d}", "dPTx", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldff1_gather_sxtw">;
411def SVLDFF1UH_GATHER_32B_OFFSETS_S : MInst<"svldff1uh_gather_[{3}]offset_{d}", "dPXx", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_sxtw">;
412
413// First-faulting load one vector (scalar base, unsigned vector offset in bytes)
414def SVLDFF1_GATHER_64B_OFFSETS_U : MInst<"svldff1_gather_[{3}]offset[_{d}]", "dPcu", "lUld", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldff1_gather">;
415def SVLDFF1SB_GATHER_64B_OFFSETS_U : MInst<"svldff1sb_gather_[{3}]offset_{d}", "dPSu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldff1_gather">;
416def SVLDFF1UB_GATHER_64B_OFFSETS_U : MInst<"svldff1ub_gather_[{3}]offset_{d}", "dPWu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldff1_gather">;
417def SVLDFF1SH_GATHER_64B_OFFSETS_U : MInst<"svldff1sh_gather_[{3}]offset_{d}", "dPTu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldff1_gather">;
418def SVLDFF1UH_GATHER_64B_OFFSETS_U : MInst<"svldff1uh_gather_[{3}]offset_{d}", "dPXu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather">;
419def SVLDFF1SW_GATHER_64B_OFFSETS_U : MInst<"svldff1sw_gather_[{3}]offset_{d}", "dPUu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ldff1_gather">;
420def SVLDFF1UW_GATHER_64B_OFFSETS_U : MInst<"svldff1uw_gather_[{3}]offset_{d}", "dPYu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1_gather">;
421
422def SVLDFF1_GATHER_32B_OFFSETS_U : MInst<"svldff1_gather_[{3}]offset[_{d}]", "dPcu", "iUif", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldff1_gather_uxtw">;
423def SVLDFF1SB_GATHER_32B_OFFSETS_U : MInst<"svldff1sb_gather_[{3}]offset_{d}", "dPSu", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldff1_gather_uxtw">;
424def SVLDFF1UB_GATHER_32B_OFFSETS_U : MInst<"svldff1ub_gather_[{3}]offset_{d}", "dPWu", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldff1_gather_uxtw">;
425def SVLDFF1SH_GATHER_32B_OFFSETS_U : MInst<"svldff1sh_gather_[{3}]offset_{d}", "dPTu", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldff1_gather_uxtw">;
426def SVLDFF1UH_GATHER_32B_OFFSETS_U : MInst<"svldff1uh_gather_[{3}]offset_{d}", "dPXu", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_uxtw">;
427
428// First-faulting load one vector (vector base, signed scalar offset in bytes)
429def SVLDFF1_GATHER_OFFSET_S : MInst<"svldff1_gather[_{2}base]_offset_{d}", "dPul", "ilUiUlfd", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldff1_gather_scalar_offset">;
430def SVLDFF1SB_GATHER_OFFSET_S : MInst<"svldff1sb_gather[_{2}base]_offset_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldff1_gather_scalar_offset">;
431def SVLDFF1UB_GATHER_OFFSET_S : MInst<"svldff1ub_gather[_{2}base]_offset_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldff1_gather_scalar_offset">;
432def SVLDFF1SH_GATHER_OFFSET_S : MInst<"svldff1sh_gather[_{2}base]_offset_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldff1_gather_scalar_offset">;
433def SVLDFF1UH_GATHER_OFFSET_S : MInst<"svldff1uh_gather[_{2}base]_offset_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_scalar_offset">;
434def SVLDFF1SW_GATHER_OFFSET_S : MInst<"svldff1sw_gather[_{2}base]_offset_{d}", "dPul", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ldff1_gather_scalar_offset">;
435def SVLDFF1UW_GATHER_OFFSET_S : MInst<"svldff1uw_gather[_{2}base]_offset_{d}", "dPul", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1_gather_scalar_offset">;
436
437// First-faulting load one vector (scalar base, signed vector index)
438def SVLDFF1_GATHER_64B_INDICES_S : MInst<"svldff1_gather_[{3}]index[_{d}]", "dPcx", "lUld", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldff1_gather_index">;
439def SVLDFF1SH_GATHER_64B_INDICES_S : MInst<"svldff1sh_gather_[{3}]index_{d}", "dPTx", "lUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldff1_gather_index">;
440def SVLDFF1UH_GATHER_64B_INDICES_S : MInst<"svldff1uh_gather_[{3}]index_{d}", "dPXx", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_index">;
441def SVLDFF1SW_GATHER_64B_INDICES_S : MInst<"svldff1sw_gather_[{3}]index_{d}", "dPUx", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ldff1_gather_index">;
442def SVLDFF1UW_GATHER_64B_INDICES_S : MInst<"svldff1uw_gather_[{3}]index_{d}", "dPYx", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1_gather_index">;
443
444def SVLDFF1_GATHER_32B_INDICES_S : MInst<"svldff1_gather_[{3}]index[_{d}]", "dPcx", "iUif", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldff1_gather_sxtw_index">;
445def SVLDFF1SH_GATHER_32B_INDICES_S : MInst<"svldff1sh_gather_[{3}]index_{d}", "dPTx", "iUi", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldff1_gather_sxtw_index">;
446def SVLDFF1UH_GATHER_32B_INDICES_S : MInst<"svldff1uh_gather_[{3}]index_{d}", "dPXx", "iUi", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_sxtw_index">;
447
448// First-faulting load one vector (scalar base, unsigned vector index)
449def SVLDFF1_GATHER_64B_INDICES_U : MInst<"svldff1_gather_[{3}]index[_{d}]", "dPcu", "lUld", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldff1_gather_index">;
450def SVLDFF1SH_GATHER_64B_INDICES_U : MInst<"svldff1sh_gather_[{3}]index_{d}", "dPTu", "lUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldff1_gather_index">;
451def SVLDFF1UH_GATHER_64B_INDICES_U : MInst<"svldff1uh_gather_[{3}]index_{d}", "dPXu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_index">;
452def SVLDFF1SW_GATHER_64B_INDICES_U : MInst<"svldff1sw_gather_[{3}]index_{d}", "dPUu", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ldff1_gather_index">;
453def SVLDFF1UW_GATHER_64B_INDICES_U : MInst<"svldff1uw_gather_[{3}]index_{d}", "dPYu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1_gather_index">;
454
455def SVLDFF1_GATHER_32B_INDICES_U : MInst<"svldff1_gather_[{3}]index[_{d}]", "dPcu", "iUif", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldff1_gather_uxtw_index">;
456def SVLDFF1SH_GATHER_32B_INDICES_U : MInst<"svldff1sh_gather_[{3}]index_{d}", "dPTu", "iUi", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldff1_gather_uxtw_index">;
457def SVLDFF1UH_GATHER_32B_INDICES_U : MInst<"svldff1uh_gather_[{3}]index_{d}", "dPXu", "iUi", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_uxtw_index">;
458
459// First-faulting load one vector (vector base, signed scalar index)
460def SVLDFF1_GATHER_INDEX_S : MInst<"svldff1_gather[_{2}base]_index_{d}", "dPul", "ilUiUlfd", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldff1_gather_scalar_offset">;
461def SVLDFF1SH_GATHER_INDEX_S : MInst<"svldff1sh_gather[_{2}base]_index_{d}", "dPul", "ilUiUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldff1_gather_scalar_offset">;
462def SVLDFF1UH_GATHER_INDEX_S : MInst<"svldff1uh_gather[_{2}base]_index_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_scalar_offset">;
463def SVLDFF1SW_GATHER_INDEX_S : MInst<"svldff1sw_gather[_{2}base]_index_{d}", "dPul", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ldff1_gather_scalar_offset">;
464def SVLDFF1UW_GATHER_INDEX_S : MInst<"svldff1uw_gather[_{2}base]_index_{d}", "dPul", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1_gather_scalar_offset">;
465
466// Non-faulting load one vector (scalar base)
467def SVLDNF1 : MInst<"svldnf1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnf1">;
468def SVLDNF1SB : MInst<"svldnf1sb_{d}", "dPS", "silUsUiUl", [IsLoad], MemEltTyInt8, "aarch64_sve_ldnf1">;
469def SVLDNF1UB : MInst<"svldnf1ub_{d}", "dPW", "silUsUiUl", [IsLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldnf1">;
470def SVLDNF1SH : MInst<"svldnf1sh_{d}", "dPT", "ilUiUl", [IsLoad], MemEltTyInt16, "aarch64_sve_ldnf1">;
471def SVLDNF1UH : MInst<"svldnf1uh_{d}", "dPX", "ilUiUl", [IsLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnf1">;
472def SVLDNF1SW : MInst<"svldnf1sw_{d}", "dPU", "lUl", [IsLoad], MemEltTyInt32, "aarch64_sve_ldnf1">;
473def SVLDNF1UW : MInst<"svldnf1uw_{d}", "dPY", "lUl", [IsLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnf1">;
474
475// Non-faulting load one vector (scalar base, VL displacement)
476def SVLDNF1_VNUM : MInst<"svldnf1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfd", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnf1">;
477def SVLDNF1SB_VNUM : MInst<"svldnf1sb_vnum_{d}", "dPSl", "silUsUiUl", [IsLoad], MemEltTyInt8, "aarch64_sve_ldnf1">;
478def SVLDNF1UB_VNUM : MInst<"svldnf1ub_vnum_{d}", "dPWl", "silUsUiUl", [IsLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldnf1">;
479def SVLDNF1SH_VNUM : MInst<"svldnf1sh_vnum_{d}", "dPTl", "ilUiUl", [IsLoad], MemEltTyInt16, "aarch64_sve_ldnf1">;
480def SVLDNF1UH_VNUM : MInst<"svldnf1uh_vnum_{d}", "dPXl", "ilUiUl", [IsLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnf1">;
481def SVLDNF1SW_VNUM : MInst<"svldnf1sw_vnum_{d}", "dPUl", "lUl", [IsLoad], MemEltTyInt32, "aarch64_sve_ldnf1">;
482def SVLDNF1UW_VNUM : MInst<"svldnf1uw_vnum_{d}", "dPYl", "lUl", [IsLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnf1">;
483
484let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
485 def SVLDNF1_BF : MInst<"svldnf1[_{2}]", "dPc", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnf1">;
486 def SVLDNF1_VNUM_BF : MInst<"svldnf1_vnum[_{2}]", "dPcl", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnf1">;
487}
488
489// Load one vector, unextended load, non-temporal (scalar base)
490def SVLDNT1 : MInst<"svldnt1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnt1">;
491
492// Load one vector, unextended load, non-temporal (scalar base, VL displacement)
493def SVLDNT1_VNUM : MInst<"svldnt1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfd", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnt1">;
494
495let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
496 def SVLDNT1_BF : MInst<"svldnt1[_{2}]", "dPc", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnt1">;
497 def SVLDNT1_VNUM_BF : MInst<"svldnt1_vnum[_{2}]", "dPcl", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnt1">;
498}
499
500// Load one quadword and replicate (scalar base)
501def SVLD1RQ : SInst<"svld1rq[_{2}]", "dPc", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ld1rq">;
502
503let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
504 def SVLD1RQ_BF : SInst<"svld1rq[_{2}]", "dPc", "b", MergeNone, "aarch64_sve_ld1rq">;
505}
506
507multiclass StructLoad<string name, string proto, string i> {
508 def : SInst<name, proto, "csilUcUsUiUlhfd", MergeNone, i, [IsStructLoad]>;
509 let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
510 def: SInst<name, proto, "b", MergeNone, i, [IsStructLoad]>;
511 }
512}
513
514// Load N-element structure into N vectors (scalar base)
515defm SVLD2 : StructLoad<"svld2[_{2}]", "2Pc", "aarch64_sve_ld2">;
516defm SVLD3 : StructLoad<"svld3[_{2}]", "3Pc", "aarch64_sve_ld3">;
517defm SVLD4 : StructLoad<"svld4[_{2}]", "4Pc", "aarch64_sve_ld4">;
518
519// Load N-element structure into N vectors (scalar base, VL displacement)
520defm SVLD2_VNUM : StructLoad<"svld2_vnum[_{2}]", "2Pcl", "aarch64_sve_ld2">;
521defm SVLD3_VNUM : StructLoad<"svld3_vnum[_{2}]", "3Pcl", "aarch64_sve_ld3">;
522defm SVLD4_VNUM : StructLoad<"svld4_vnum[_{2}]", "4Pcl", "aarch64_sve_ld4">;
523
524// Load one octoword and replicate (scalar base)
525let ArchGuard = "defined(__ARM_FEATURE_SVE_MATMUL_FP64)" in {
526 def SVLD1RO : SInst<"svld1ro[_{2}]", "dPc", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ld1ro">;
527}
528let ArchGuard = "defined(__ARM_FEATURE_SVE_MATMUL_FP64) && defined(__ARM_FEATURE_SVE_BF16)" in {
529 def SVLD1RO_BF16 : SInst<"svld1ro[_{2}]", "dPc", "b", MergeNone, "aarch64_sve_ld1ro">;
530}
531
532let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
533 def SVBFDOT : SInst<"svbfdot[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfdot", [IsOverloadNone]>;
534 def SVBFMLALB : SInst<"svbfmlalb[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfmlalb", [IsOverloadNone]>;
535 def SVBFMLALT : SInst<"svbfmlalt[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfmlalt", [IsOverloadNone]>;
536 def SVBFMMLA : SInst<"svbfmmla[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfmmla", [IsOverloadNone]>;
537 def SVBFDOT_N : SInst<"svbfdot[_n_{0}]", "MMda", "b", MergeNone, "aarch64_sve_bfdot", [IsOverloadNone]>;
538 def SVBFMLAL_N : SInst<"svbfmlalb[_n_{0}]", "MMda", "b", MergeNone, "aarch64_sve_bfmlalb", [IsOverloadNone]>;
539 def SVBFMLALT_N : SInst<"svbfmlalt[_n_{0}]", "MMda", "b", MergeNone, "aarch64_sve_bfmlalt", [IsOverloadNone]>;
540 def SVBFDOT_LANE : SInst<"svbfdot_lane[_{0}]", "MMddn", "b", MergeNone, "aarch64_sve_bfdot_lane", [IsOverloadNone], [ImmCheck<3, ImmCheck0_3>]>;
541 def SVBFMLALB_LANE : SInst<"svbfmlalb_lane[_{0}]", "MMddn", "b", MergeNone, "aarch64_sve_bfmlalb_lane", [IsOverloadNone], [ImmCheck<3, ImmCheck0_7>]>;
542 def SVBFMLALT_LANE : SInst<"svbfmlalt_lane[_{0}]", "MMddn", "b", MergeNone, "aarch64_sve_bfmlalt_lane", [IsOverloadNone], [ImmCheck<3, ImmCheck0_7>]>;
543}
544
545////////////////////////////////////////////////////////////////////////////////
546// Stores
547
548// Store one vector (scalar base)
549def SVST1 : MInst<"svst1[_{d}]", "vPpd", "csilUcUsUiUlhfd", [IsStore], MemEltTyDefault, "aarch64_sve_st1">;
550def SVST1B_S : MInst<"svst1b[_{d}]", "vPAd", "sil", [IsStore], MemEltTyInt8, "aarch64_sve_st1">;
551def SVST1B_U : MInst<"svst1b[_{d}]", "vPEd", "UsUiUl", [IsStore], MemEltTyInt8, "aarch64_sve_st1">;
552def SVST1H_S : MInst<"svst1h[_{d}]", "vPBd", "il", [IsStore], MemEltTyInt16, "aarch64_sve_st1">;
553def SVST1H_U : MInst<"svst1h[_{d}]", "vPFd", "UiUl", [IsStore], MemEltTyInt16, "aarch64_sve_st1">;
554def SVST1W_S : MInst<"svst1w[_{d}]", "vPCd", "l", [IsStore], MemEltTyInt32, "aarch64_sve_st1">;
555def SVST1W_U : MInst<"svst1w[_{d}]", "vPGd", "Ul", [IsStore], MemEltTyInt32, "aarch64_sve_st1">;
556
557// Store one vector (scalar base, VL displacement)
558def SVST1_VNUM : MInst<"svst1_vnum[_{d}]", "vPpld", "csilUcUsUiUlhfd", [IsStore], MemEltTyDefault, "aarch64_sve_st1">;
559def SVST1B_VNUM_S : MInst<"svst1b_vnum[_{d}]", "vPAld", "sil", [IsStore], MemEltTyInt8, "aarch64_sve_st1">;
560def SVST1B_VNUM_U : MInst<"svst1b_vnum[_{d}]", "vPEld", "UsUiUl", [IsStore], MemEltTyInt8, "aarch64_sve_st1">;
561def SVST1H_VNUM_S : MInst<"svst1h_vnum[_{d}]", "vPBld", "il", [IsStore], MemEltTyInt16, "aarch64_sve_st1">;
562def SVST1H_VNUM_U : MInst<"svst1h_vnum[_{d}]", "vPFld", "UiUl", [IsStore], MemEltTyInt16, "aarch64_sve_st1">;
563def SVST1W_VNUM_S : MInst<"svst1w_vnum[_{d}]", "vPCld", "l", [IsStore], MemEltTyInt32, "aarch64_sve_st1">;
564def SVST1W_VNUM_U : MInst<"svst1w_vnum[_{d}]", "vPGld", "Ul", [IsStore], MemEltTyInt32, "aarch64_sve_st1">;
565
566let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
567 def SVST1_BF : MInst<"svst1[_{d}]", "vPpd", "b", [IsStore], MemEltTyDefault, "aarch64_sve_st1">;
568 def SVST1_VNUM_BF : MInst<"svst1_vnum[_{d}]", "vPpld", "b", [IsStore], MemEltTyDefault, "aarch64_sve_st1">;
569}
570
571// Store one vector (vector base)
572def SVST1_SCATTER_BASES_U : MInst<"svst1_scatter[_{2}base_{d}]", "vPud", "ilUiUlfd", [IsScatterStore], MemEltTyDefault, "aarch64_sve_st1_scatter_scalar_offset">;
573def SVST1B_SCATTER_BASES_U : MInst<"svst1b_scatter[_{2}base_{d}]", "vPud", "ilUiUl", [IsScatterStore], MemEltTyInt8, "aarch64_sve_st1_scatter_scalar_offset">;
574def SVST1H_SCATTER_BASES_U : MInst<"svst1h_scatter[_{2}base_{d}]", "vPud", "ilUiUl", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_scalar_offset">;
575def SVST1W_SCATTER_BASES_U : MInst<"svst1w_scatter[_{2}base_{d}]", "vPud", "lUl", [IsScatterStore], MemEltTyInt32, "aarch64_sve_st1_scatter_scalar_offset">;
576
577// Store one vector (scalar base, signed vector offset in bytes)
578def SVST1_SCATTER_64B_OFFSETS_S : MInst<"svst1_scatter_[{3}]offset[_{d}]", "vPpxd", "lUld", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_st1_scatter">;
579def SVST1B_SCATTER_64B_OFFSETS_SS : MInst<"svst1b_scatter_[{3}]offset[_{d}]", "vPAxd", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter">;
580def SVST1B_SCATTER_64B_OFFSETS_SU : MInst<"svst1b_scatter_[{3}]offset[_{d}]", "vPExd", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter">;
581def SVST1H_SCATTER_64B_OFFSETS_SS : MInst<"svst1h_scatter_[{3}]offset[_{d}]", "vPBxd", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter">;
582def SVST1H_SCATTER_64B_OFFSETS_SU : MInst<"svst1h_scatter_[{3}]offset[_{d}]", "vPFxd", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter">;
583def SVST1W_SCATTER_64B_OFFSETS_SS : MInst<"svst1w_scatter_[{3}]offset[_{d}]", "vPCxd", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_st1_scatter">;
584def SVST1W_SCATTER_64B_OFFSETS_SU : MInst<"svst1w_scatter_[{3}]offset[_{d}]", "vPGxd", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_st1_scatter">;
585
586def SVST1_SCATTER_32B_OFFSETS_S : MInst<"svst1_scatter_[{3}]offset[_{d}]", "vPpxd", "iUif", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_st1_scatter_sxtw">;
587def SVST1B_SCATTER_32B_OFFSETS_SS : MInst<"svst1b_scatter_[{3}]offset[_{d}]", "vPAxd", "i", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter_sxtw">;
588def SVST1B_SCATTER_32B_OFFSETS_SU : MInst<"svst1b_scatter_[{3}]offset[_{d}]", "vPExd", "Ui", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter_sxtw">;
589def SVST1H_SCATTER_32B_OFFSETS_SS : MInst<"svst1h_scatter_[{3}]offset[_{d}]", "vPBxd", "i", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter_sxtw">;
590def SVST1H_SCATTER_32B_OFFSETS_SU : MInst<"svst1h_scatter_[{3}]offset[_{d}]", "vPFxd", "Ui", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter_sxtw">;
591
592// Store one vector (scalar base, unsigned vector offset in bytes)
593def SVST1_SCATTER_64B_OFFSETS_U : MInst<"svst1_scatter_[{3}]offset[_{d}]", "vPpud", "lUld", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_st1_scatter">;
594def SVST1B_SCATTER_64B_OFFSETS_US : MInst<"svst1b_scatter_[{3}]offset[_{d}]", "vPAud", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter">;
595def SVST1B_SCATTER_64B_OFFSETS_UU : MInst<"svst1b_scatter_[{3}]offset[_{d}]", "vPEud", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter">;
596def SVST1H_SCATTER_64B_OFFSETS_US : MInst<"svst1h_scatter_[{3}]offset[_{d}]", "vPBud", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter">;
597def SVST1H_SCATTER_64B_OFFSETS_UU : MInst<"svst1h_scatter_[{3}]offset[_{d}]", "vPFud", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter">;
598def SVST1W_SCATTER_64B_OFFSETS_US : MInst<"svst1w_scatter_[{3}]offset[_{d}]", "vPCud", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_st1_scatter">;
599def SVST1W_SCATTER_64B_OFFSETS_UU : MInst<"svst1w_scatter_[{3}]offset[_{d}]", "vPGud", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_st1_scatter">;
600
601def SVST1_SCATTER_32B_OFFSETS_U : MInst<"svst1_scatter_[{3}]offset[_{d}]", "vPpud", "iUif", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_st1_scatter_uxtw">;
602def SVST1B_SCATTER_32B_OFFSETS_US : MInst<"svst1b_scatter_[{3}]offset[_{d}]", "vPAud", "i", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter_uxtw">;
603def SVST1B_SCATTER_32B_OFFSETS_UU : MInst<"svst1b_scatter_[{3}]offset[_{d}]", "vPEud", "Ui", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter_uxtw">;
604def SVST1H_SCATTER_32B_OFFSETS_US : MInst<"svst1h_scatter_[{3}]offset[_{d}]", "vPBud", "i", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter_uxtw">;
605def SVST1H_SCATTER_32B_OFFSETS_UU : MInst<"svst1h_scatter_[{3}]offset[_{d}]", "vPFud", "Ui", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter_uxtw">;
606
607// Store one vector (vector base, signed scalar offset in bytes)
608def SVST1_SCATTER_OFFSET_S : MInst<"svst1_scatter[_{2}base]_offset[_{d}]", "vPuld", "ilUiUlfd", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_st1_scatter_scalar_offset">;
609def SVST1B_SCATTER_OFFSET_S : MInst<"svst1b_scatter[_{2}base]_offset[_{d}]", "vPuld", "ilUiUl", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter_scalar_offset">;
610def SVST1H_SCATTER_OFFSET_S : MInst<"svst1h_scatter[_{2}base]_offset[_{d}]", "vPuld", "ilUiUl", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter_scalar_offset">;
611def SVST1W_SCATTER_OFFSET_S : MInst<"svst1w_scatter[_{2}base]_offset[_{d}]", "vPuld", "lUl", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_st1_scatter_scalar_offset">;
612
613// Store one vector (scalar base, signed vector index)
614def SVST1_SCATTER_64B_INDICES_S : MInst<"svst1_scatter_[{3}]index[_{d}]", "vPpxd", "lUld", [IsScatterStore], MemEltTyDefault, "aarch64_sve_st1_scatter_index">;
615def SVST1H_SCATTER_64B_INDICES_SS : MInst<"svst1h_scatter_[{3}]index[_{d}]", "vPBxd", "l", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_index">;
616def SVST1H_SCATTER_64B_INDICES_SU : MInst<"svst1h_scatter_[{3}]index[_{d}]", "vPFxd", "Ul", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_index">;
617def SVST1W_SCATTER_64B_INDICES_SS : MInst<"svst1w_scatter_[{3}]index[_{d}]", "vPCxd", "l", [IsScatterStore], MemEltTyInt32, "aarch64_sve_st1_scatter_index">;
618def SVST1W_SCATTER_64B_INDICES_SU : MInst<"svst1w_scatter_[{3}]index[_{d}]", "vPGxd", "Ul", [IsScatterStore], MemEltTyInt32, "aarch64_sve_st1_scatter_index">;
619
620def SVST1_SCATTER_32B_INDICES_S : MInst<"svst1_scatter_[{3}]index[_{d}]", "vPpxd", "iUif", [IsScatterStore], MemEltTyDefault, "aarch64_sve_st1_scatter_sxtw_index">;
621def SVST1H_SCATTER_32B_INDICES_SS : MInst<"svst1h_scatter_[{3}]index[_{d}]", "vPBxd", "i", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_sxtw_index">;
622def SVST1H_SCATTER_32B_INDICES_SU : MInst<"svst1h_scatter_[{3}]index[_{d}]", "vPFxd", "Ui", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_sxtw_index">;
623
624// Store one vector (scalar base, unsigned vector index)
625def SVST1_SCATTER_64B_INDICES_U : MInst<"svst1_scatter_[{3}]index[_{d}]", "vPpud", "lUld", [IsScatterStore], MemEltTyDefault, "aarch64_sve_st1_scatter_index">;
626def SVST1H_SCATTER_64B_INDICES_US : MInst<"svst1h_scatter_[{3}]index[_{d}]", "vPBud", "l", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_index">;
627def SVST1H_SCATTER_64B_INDICES_UU : MInst<"svst1h_scatter_[{3}]index[_{d}]", "vPFud", "Ul", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_index">;
628def SVST1W_SCATTER_64B_INDICES_US : MInst<"svst1w_scatter_[{3}]index[_{d}]", "vPCud", "l", [IsScatterStore], MemEltTyInt32, "aarch64_sve_st1_scatter_index">;
629def SVST1W_SCATTER_64B_INDICES_UU : MInst<"svst1w_scatter_[{3}]index[_{d}]", "vPGud", "Ul", [IsScatterStore], MemEltTyInt32, "aarch64_sve_st1_scatter_index">;
630
631def SVST1_SCATTER_32B_INDICES_U : MInst<"svst1_scatter_[{3}]index[_{d}]", "vPpud", "iUif", [IsScatterStore], MemEltTyDefault, "aarch64_sve_st1_scatter_uxtw_index">;
632def SVST1H_SCATTER_32B_INDICES_US : MInst<"svst1h_scatter_[{3}]index[_{d}]", "vPBud", "i", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_uxtw_index">;
633def SVST1H_SCATTER_32B_INDICES_UU : MInst<"svst1h_scatter_[{3}]index[_{d}]", "vPFud", "Ui", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_uxtw_index">;
634
635// Store one vector (vector base, signed scalar index)
636def SVST1_SCATTER_INDEX_S : MInst<"svst1_scatter[_{2}base]_index[_{d}]", "vPuld", "ilUiUlfd", [IsScatterStore], MemEltTyDefault, "aarch64_sve_st1_scatter_scalar_offset">;
637def SVST1H_SCATTER_INDEX_S : MInst<"svst1h_scatter[_{2}base]_index[_{d}]", "vPuld", "ilUiUl", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_scalar_offset">;
638def SVST1W_SCATTER_INDEX_S : MInst<"svst1w_scatter[_{2}base]_index[_{d}]", "vPuld", "lUl", [IsScatterStore], MemEltTyInt32, "aarch64_sve_st1_scatter_scalar_offset">;
639
640multiclass StructStore<string name, string proto, string i> {
641 def : SInst<name, proto, "csilUcUsUiUlhfd", MergeNone, i, [IsStructStore]>;
642 let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
643 def: SInst<name, proto, "b", MergeNone, i, [IsStructStore]>;
644 }
645}
646// Store N vectors into N-element structure (scalar base)
647defm SVST2 : StructStore<"svst2[_{d}]", "vPp2", "aarch64_sve_st2">;
648defm SVST3 : StructStore<"svst3[_{d}]", "vPp3", "aarch64_sve_st3">;
649defm SVST4 : StructStore<"svst4[_{d}]", "vPp4", "aarch64_sve_st4">;
650
651// Store N vectors into N-element structure (scalar base, VL displacement)
652defm SVST2_VNUM : StructStore<"svst2_vnum[_{d}]", "vPpl2", "aarch64_sve_st2">;
653defm SVST3_VNUM : StructStore<"svst3_vnum[_{d}]", "vPpl3", "aarch64_sve_st3">;
654defm SVST4_VNUM : StructStore<"svst4_vnum[_{d}]", "vPpl4", "aarch64_sve_st4">;
655
656// Store one vector, with no truncation, non-temporal (scalar base)
657def SVSTNT1 : MInst<"svstnt1[_{d}]", "vPpd", "csilUcUsUiUlhfd", [IsStore], MemEltTyDefault, "aarch64_sve_stnt1">;
658
659// Store one vector, with no truncation, non-temporal (scalar base, VL displacement)
660def SVSTNT1_VNUM : MInst<"svstnt1_vnum[_{d}]", "vPpld", "csilUcUsUiUlhfd", [IsStore], MemEltTyDefault, "aarch64_sve_stnt1">;
661
662let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
663 def SVSTNT1_BF : MInst<"svstnt1[_{d}]", "vPpd", "b", [IsStore], MemEltTyDefault, "aarch64_sve_stnt1">;
664 def SVSTNT1_VNUM_BF : MInst<"svstnt1_vnum[_{d}]", "vPpld", "b", [IsStore], MemEltTyDefault, "aarch64_sve_stnt1">;
665}
666
667////////////////////////////////////////////////////////////////////////////////
668// Prefetches
669
670// Prefetch (Scalar base)
671def SVPRFB : MInst<"svprfb", "vPQJ", "c", [IsPrefetch], MemEltTyInt8, "aarch64_sve_prf">;
672def SVPRFH : MInst<"svprfh", "vPQJ", "s", [IsPrefetch], MemEltTyInt16, "aarch64_sve_prf">;
673def SVPRFW : MInst<"svprfw", "vPQJ", "i", [IsPrefetch], MemEltTyInt32, "aarch64_sve_prf">;
674def SVPRFD : MInst<"svprfd", "vPQJ", "l", [IsPrefetch], MemEltTyInt64, "aarch64_sve_prf">;
675
676// Prefetch (Scalar base, VL displacement)
677def SVPRFB_VNUM : MInst<"svprfb_vnum", "vPQlJ", "c", [IsPrefetch], MemEltTyInt8, "aarch64_sve_prf">;
678def SVPRFH_VNUM : MInst<"svprfh_vnum", "vPQlJ", "s", [IsPrefetch], MemEltTyInt16, "aarch64_sve_prf">;
679def SVPRFW_VNUM : MInst<"svprfw_vnum", "vPQlJ", "i", [IsPrefetch], MemEltTyInt32, "aarch64_sve_prf">;
680def SVPRFD_VNUM : MInst<"svprfd_vnum", "vPQlJ", "l", [IsPrefetch], MemEltTyInt64, "aarch64_sve_prf">;
681
682// Prefetch (Vector bases)
683def SVPRFB_GATHER_BASES : MInst<"svprfb_gather[_{2}base]", "vPdJ", "UiUl", [IsGatherPrefetch], MemEltTyInt8, "aarch64_sve_prfb_gather_scalar_offset">;
684def SVPRFH_GATHER_BASES : MInst<"svprfh_gather[_{2}base]", "vPdJ", "UiUl", [IsGatherPrefetch], MemEltTyInt16, "aarch64_sve_prfh_gather_scalar_offset">;
685def SVPRFW_GATHER_BASES : MInst<"svprfw_gather[_{2}base]", "vPdJ", "UiUl", [IsGatherPrefetch], MemEltTyInt32, "aarch64_sve_prfw_gather_scalar_offset">;
686def SVPRFD_GATHER_BASES : MInst<"svprfd_gather[_{2}base]", "vPdJ", "UiUl", [IsGatherPrefetch], MemEltTyInt64, "aarch64_sve_prfd_gather_scalar_offset">;
687
688// Prefetch (Scalar base, Vector offsets)
689def SVPRFB_GATHER_32B_OFFSETS_S : MInst<"svprfb_gather_[{3}]offset", "vPQdJ", "i", [IsGatherPrefetch], MemEltTyInt8, "aarch64_sve_prfb_gather_sxtw_index">;
690def SVPRFH_GATHER_32B_OFFSETS_S : MInst<"svprfh_gather_[{3}]index", "vPQdJ", "i", [IsGatherPrefetch], MemEltTyInt16, "aarch64_sve_prfh_gather_sxtw_index">;
691def SVPRFW_GATHER_32B_OFFSETS_S : MInst<"svprfw_gather_[{3}]index", "vPQdJ", "i", [IsGatherPrefetch], MemEltTyInt32, "aarch64_sve_prfw_gather_sxtw_index">;
692def SVPRFD_GATHER_32B_OFFSETS_S : MInst<"svprfd_gather_[{3}]index", "vPQdJ", "i", [IsGatherPrefetch], MemEltTyInt64, "aarch64_sve_prfd_gather_sxtw_index">;
693
694def SVPRFB_GATHER_64B_OFFSETS_S : MInst<"svprfb_gather_[{3}]offset", "vPQdJ", "l", [IsGatherPrefetch], MemEltTyInt8, "aarch64_sve_prfb_gather_index">;
695def SVPRFH_GATHER_64B_OFFSETS_S : MInst<"svprfh_gather_[{3}]index", "vPQdJ", "l", [IsGatherPrefetch], MemEltTyInt16, "aarch64_sve_prfh_gather_index">;
696def SVPRFW_GATHER_64B_OFFSETS_S : MInst<"svprfw_gather_[{3}]index", "vPQdJ", "l", [IsGatherPrefetch], MemEltTyInt32, "aarch64_sve_prfw_gather_index">;
697def SVPRFD_GATHER_64B_OFFSETS_S : MInst<"svprfd_gather_[{3}]index", "vPQdJ", "l", [IsGatherPrefetch], MemEltTyInt64, "aarch64_sve_prfd_gather_index">;
698
699def SVPRFB_GATHER_32B_OFFSETS_U : MInst<"svprfb_gather_[{3}]offset", "vPQdJ", "Ui", [IsGatherPrefetch], MemEltTyInt8, "aarch64_sve_prfb_gather_uxtw_index">;
700def SVPRFH_GATHER_32B_OFFSETS_U : MInst<"svprfh_gather_[{3}]index", "vPQdJ", "Ui", [IsGatherPrefetch], MemEltTyInt16, "aarch64_sve_prfh_gather_uxtw_index">;
701def SVPRFW_GATHER_32B_OFFSETS_U : MInst<"svprfw_gather_[{3}]index", "vPQdJ", "Ui", [IsGatherPrefetch], MemEltTyInt32, "aarch64_sve_prfw_gather_uxtw_index">;
702def SVPRFD_GATHER_32B_OFFSETS_U : MInst<"svprfd_gather_[{3}]index", "vPQdJ", "Ui", [IsGatherPrefetch], MemEltTyInt64, "aarch64_sve_prfd_gather_uxtw_index">;
703
704def SVPRFB_GATHER_64B_OFFSETS_U : MInst<"svprfb_gather_[{3}]offset", "vPQdJ", "Ul", [IsGatherPrefetch], MemEltTyInt8, "aarch64_sve_prfb_gather_index">;
705def SVPRFH_GATHER_64B_OFFSETS_U : MInst<"svprfh_gather_[{3}]index", "vPQdJ", "Ul", [IsGatherPrefetch], MemEltTyInt16, "aarch64_sve_prfh_gather_index">;
706def SVPRFW_GATHER_64B_OFFSETS_U : MInst<"svprfw_gather_[{3}]index", "vPQdJ", "Ul", [IsGatherPrefetch], MemEltTyInt32, "aarch64_sve_prfw_gather_index">;
707def SVPRFD_GATHER_64B_OFFSETS_U : MInst<"svprfd_gather_[{3}]index", "vPQdJ", "Ul", [IsGatherPrefetch], MemEltTyInt64, "aarch64_sve_prfd_gather_index">;
708
709// Prefetch (Vector bases, scalar offset)
710def SVPRFB_GATHER_BASES_OFFSET : MInst<"svprfb_gather[_{2}base]_offset", "vPdlJ", "UiUl", [IsGatherPrefetch], MemEltTyInt8, "aarch64_sve_prfb_gather_scalar_offset">;
711def SVPRFH_GATHER_BASES_OFFSET : MInst<"svprfh_gather[_{2}base]_index", "vPdlJ", "UiUl", [IsGatherPrefetch], MemEltTyInt16, "aarch64_sve_prfh_gather_scalar_offset">;
712def SVPRFW_GATHER_BASES_OFFSET : MInst<"svprfw_gather[_{2}base]_index", "vPdlJ", "UiUl", [IsGatherPrefetch], MemEltTyInt32, "aarch64_sve_prfw_gather_scalar_offset">;
713def SVPRFD_GATHER_BASES_OFFSET : MInst<"svprfd_gather[_{2}base]_index", "vPdlJ", "UiUl", [IsGatherPrefetch], MemEltTyInt64, "aarch64_sve_prfd_gather_scalar_offset">;
714
715////////////////////////////////////////////////////////////////////////////////
716// Address calculations
717
718def SVADRB : SInst<"svadrb[_{0}base]_[{2}]offset", "uud", "ilUiUl", MergeNone, "aarch64_sve_adrb">;
719def SVADRH : SInst<"svadrh[_{0}base]_[{2}]index", "uud", "ilUiUl", MergeNone, "aarch64_sve_adrh">;
720def SVADRW : SInst<"svadrw[_{0}base]_[{2}]index", "uud", "ilUiUl", MergeNone, "aarch64_sve_adrw">;
721def SVADRD : SInst<"svadrd[_{0}base]_[{2}]index", "uud", "ilUiUl", MergeNone, "aarch64_sve_adrd">;
722
723////////////////////////////////////////////////////////////////////////////////
724// Scalar to vector
725
726def SVDUPQ_8 : SInst<"svdupq[_n]_{d}", "dssssssssssssssss", "cUc", MergeNone>;
727def SVDUPQ_16 : SInst<"svdupq[_n]_{d}", "dssssssss", "sUsh", MergeNone>;
728let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
729 def SVDUPQ_BF16 : SInst<"svdupq[_n]_{d}", "dssssssss", "b", MergeNone>;
730}
731def SVDUPQ_32 : SInst<"svdupq[_n]_{d}", "dssss", "iUif", MergeNone>;
732def SVDUPQ_64 : SInst<"svdupq[_n]_{d}", "dss", "lUld", MergeNone>;
733
734multiclass svdup_base<string n, string p, MergeType mt, string i> {
735 def NAME : SInst<n, p, "csilUcUsUiUlhfd", mt, i>;
736 let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
737 def _BF16: SInst<n, p, "b", mt, i>;
738 }
739}
740
741defm SVDUP : svdup_base<"svdup[_n]_{d}", "ds", MergeNone, "aarch64_sve_dup_x">;
742defm SVDUP_M : svdup_base<"svdup[_n]_{d}", "ddPs", MergeOp1, "aarch64_sve_dup">;
743defm SVDUP_X : svdup_base<"svdup[_n]_{d}", "dPs", MergeAnyExp, "aarch64_sve_dup">;
744defm SVDUP_Z : svdup_base<"svdup[_n]_{d}", "dPs", MergeZeroExp, "aarch64_sve_dup">;
745
746def SVINDEX : SInst<"svindex_{d}", "dss", "csilUcUsUiUl", MergeNone, "aarch64_sve_index">;
747
748// Integer arithmetic
749
750multiclass SInstZPZ<string name, string types, string intrinsic, list<FlagType> flags=[]> {
751 def _M : SInst<name # "[_{d}]", "ddPd", types, MergeOp1, intrinsic, flags>;
752 def _X : SInst<name # "[_{d}]", "dPd", types, MergeAnyExp, intrinsic, flags>;
753 def _Z : SInst<name # "[_{d}]", "dPd", types, MergeZeroExp, intrinsic, flags>;
754}
755
756defm SVABS : SInstZPZ<"svabs", "csil", "aarch64_sve_abs">;
757defm SVNEG : SInstZPZ<"svneg", "csil", "aarch64_sve_neg">;
758
759//------------------------------------------------------------------------------
760
761multiclass SInstZPZZ<string name, string types, string intrinsic, list<FlagType> flags=[]> {
762 def _M : SInst<name # "[_{d}]", "dPdd", types, MergeOp1, intrinsic, flags>;
763 def _X : SInst<name # "[_{d}]", "dPdd", types, MergeAny, intrinsic, flags>;
764 def _Z : SInst<name # "[_{d}]", "dPdd", types, MergeZero, intrinsic, flags>;
765
766 def _N_M : SInst<name # "[_n_{d}]", "dPda", types, MergeOp1, intrinsic, flags>;
767 def _N_X : SInst<name # "[_n_{d}]", "dPda", types, MergeAny, intrinsic, flags>;
768 def _N_Z : SInst<name # "[_n_{d}]", "dPda", types, MergeZero, intrinsic, flags>;
769}
770
771defm SVABD_S : SInstZPZZ<"svabd", "csil", "aarch64_sve_sabd">;
772defm SVABD_U : SInstZPZZ<"svabd", "UcUsUiUl", "aarch64_sve_uabd">;
773defm SVADD : SInstZPZZ<"svadd", "csilUcUsUiUl", "aarch64_sve_add">;
774defm SVDIV_S : SInstZPZZ<"svdiv", "il", "aarch64_sve_sdiv">;
775defm SVDIV_U : SInstZPZZ<"svdiv", "UiUl", "aarch64_sve_udiv">;
776defm SVDIVR_S : SInstZPZZ<"svdivr", "il", "aarch64_sve_sdivr">;
777defm SVDIVR_U : SInstZPZZ<"svdivr", "UiUl", "aarch64_sve_udivr">;
778defm SVMAX_S : SInstZPZZ<"svmax", "csil", "aarch64_sve_smax">;
779defm SVMAX_U : SInstZPZZ<"svmax", "UcUsUiUl", "aarch64_sve_umax">;
780defm SVMIN_S : SInstZPZZ<"svmin", "csil", "aarch64_sve_smin">;
781defm SVMIN_U : SInstZPZZ<"svmin", "UcUsUiUl", "aarch64_sve_umin">;
782defm SVMUL : SInstZPZZ<"svmul", "csilUcUsUiUl", "aarch64_sve_mul">;
783defm SVMULH_S : SInstZPZZ<"svmulh", "csil", "aarch64_sve_smulh">;
784defm SVMULH_U : SInstZPZZ<"svmulh", "UcUsUiUl", "aarch64_sve_umulh">;
785defm SVSUB : SInstZPZZ<"svsub", "csilUcUsUiUl", "aarch64_sve_sub">;
786defm SVSUBR : SInstZPZZ<"svsubr", "csilUcUsUiUl", "aarch64_sve_subr">;
787
788//------------------------------------------------------------------------------
789
790multiclass SInstZPZZZ<string name, string types, string intrinsic, list<FlagType> flags=[]> {
791 def _M : SInst<name # "[_{d}]", "dPddd", types, MergeOp1, intrinsic, flags>;
792 def _X : SInst<name # "[_{d}]", "dPddd", types, MergeAny, intrinsic, flags>;
793 def _Z : SInst<name # "[_{d}]", "dPddd", types, MergeZero, intrinsic, flags>;
794
795 def _N_M : SInst<name # "[_n_{d}]", "dPdda", types, MergeOp1, intrinsic, flags>;
796 def _N_X : SInst<name # "[_n_{d}]", "dPdda", types, MergeAny, intrinsic, flags>;
797 def _N_Z : SInst<name # "[_n_{d}]", "dPdda", types, MergeZero, intrinsic, flags>;
798}
799
800defm SVMAD : SInstZPZZZ<"svmad", "csilUcUsUiUl", "aarch64_sve_mad">;
801defm SVMLA : SInstZPZZZ<"svmla", "csilUcUsUiUl", "aarch64_sve_mla">;
802defm SVMLS : SInstZPZZZ<"svmls", "csilUcUsUiUl", "aarch64_sve_mls">;
803defm SVMSB : SInstZPZZZ<"svmsb", "csilUcUsUiUl", "aarch64_sve_msb">;
804
805//------------------------------------------------------------------------------
806
807def SVDOT_S : SInst<"svdot[_{0}]", "ddqq", "il", MergeNone, "aarch64_sve_sdot">;
808def SVDOT_U : SInst<"svdot[_{0}]", "ddqq", "UiUl", MergeNone, "aarch64_sve_udot">;
809def SVQADD_S : SInst<"svqadd[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqadd_x">;
810def SVQADD_U : SInst<"svqadd[_{d}]", "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_uqadd_x">;
811def SVQSUB_S : SInst<"svqsub[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqsub_x">;
812def SVQSUB_U : SInst<"svqsub[_{d}]", "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_uqsub_x">;
813
814def SVDOT_N_S : SInst<"svdot[_n_{0}]", "ddqr", "il", MergeNone, "aarch64_sve_sdot">;
815def SVDOT_N_U : SInst<"svdot[_n_{0}]", "ddqr", "UiUl", MergeNone, "aarch64_sve_udot">;
816def SVQADD_N_S : SInst<"svqadd[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqadd_x">;
817def SVQADD_N_U : SInst<"svqadd[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_uqadd_x">;
818def SVQSUB_N_S : SInst<"svqsub[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqsub_x">;
819def SVQSUB_N_U : SInst<"svqsub[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_uqsub_x">;
820
821def SVDOT_LANE_S : SInst<"svdot_lane[_{d}]", "ddqqi", "il", MergeNone, "aarch64_sve_sdot_lane", [], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>;
822def SVDOT_LANE_U : SInst<"svdot_lane[_{d}]", "ddqqi", "UiUl", MergeNone, "aarch64_sve_udot_lane", [], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>;
823
824////////////////////////////////////////////////////////////////////////////////
825// Logical operations
826
827defm SVAND : SInstZPZZ<"svand", "csilUcUsUiUl", "aarch64_sve_and">;
828defm SVBIC : SInstZPZZ<"svbic", "csilUcUsUiUl", "aarch64_sve_bic">;
829defm SVEOR : SInstZPZZ<"sveor", "csilUcUsUiUl", "aarch64_sve_eor">;
830defm SVORR : SInstZPZZ<"svorr", "csilUcUsUiUl", "aarch64_sve_orr">;
831
832defm SVCNOT : SInstZPZ<"svcnot", "csilUcUsUiUl", "aarch64_sve_cnot">;
833defm SVNOT : SInstZPZ<"svnot", "csilUcUsUiUl", "aarch64_sve_not">;
834
835////////////////////////////////////////////////////////////////////////////////
836// Shifts
837
838multiclass SInst_SHIFT<string name, string intrinsic, string ts, string wide_ts> {
839 def _M : SInst<name # "[_{d}]", "dPdu", ts, MergeOp1, intrinsic>;
840 def _X : SInst<name # "[_{d}]", "dPdu", ts, MergeAny, intrinsic>;
841 def _Z : SInst<name # "[_{d}]", "dPdu", ts, MergeZero, intrinsic>;
842
843 def _N_M : SInst<name # "[_n_{d}]", "dPdL", ts, MergeOp1, intrinsic>;
844 def _N_X : SInst<name # "[_n_{d}]", "dPdL", ts, MergeAny, intrinsic>;
845 def _N_Z : SInst<name # "[_n_{d}]", "dPdL", ts, MergeZero, intrinsic>;
846
847 def _WIDE_M : SInst<name # _wide # "[_{d}]", "dPdg", wide_ts, MergeOp1, intrinsic # _wide>;
848 def _WIDE_X : SInst<name # _wide # "[_{d}]", "dPdg", wide_ts, MergeAny, intrinsic # _wide>;
849 def _WIDE_Z : SInst<name # _wide # "[_{d}]", "dPdg", wide_ts, MergeZero, intrinsic # _wide>;
850
851 def _WIDE_N_M : SInst<name # _wide # "[_n_{d}]", "dPdf", wide_ts, MergeOp1, intrinsic # _wide>;
852 def _WIDE_N_X : SInst<name # _wide # "[_n_{d}]", "dPdf", wide_ts, MergeAny, intrinsic # _wide>;
853 def _WIDE_N_Z : SInst<name # _wide # "[_n_{d}]", "dPdf", wide_ts, MergeZero, intrinsic # _wide>;
854}
855
856defm SVASR : SInst_SHIFT<"svasr", "aarch64_sve_asr", "csil", "csi">;
857defm SVLSL : SInst_SHIFT<"svlsl", "aarch64_sve_lsl", "csilUcUsUiUl", "csiUcUsUi">;
858defm SVLSR : SInst_SHIFT<"svlsr", "aarch64_sve_lsr", "UcUsUiUl", "UcUsUi">;
859
860def SVASRD_M : SInst<"svasrd[_n_{d}]", "dPdi", "csil", MergeOp1, "aarch64_sve_asrd", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>;
861def SVASRD_X : SInst<"svasrd[_n_{d}]", "dPdi", "csil", MergeAny, "aarch64_sve_asrd", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>;
862def SVASRD_Z : SInst<"svasrd[_n_{d}]", "dPdi", "csil", MergeZero, "aarch64_sve_asrd", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>;
863
864def SVINSR : SInst<"svinsr[_n_{d}]", "dds", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_insr">;
865let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
866 def SVINSR_BF16 : SInst<"svinsr[_n_{d}]", "dds", "b", MergeNone, "aarch64_sve_insr">;
867}
868
869////////////////////////////////////////////////////////////////////////////////
870// Integer reductions
871
872def SVADDV_S : SInst<"svaddv[_{d}]", "lPd", "csil", MergeNone, "aarch64_sve_saddv">;
873def SVADDV_U : SInst<"svaddv[_{d}]", "nPd", "UcUsUiUl", MergeNone, "aarch64_sve_uaddv">;
874def SVANDV : SInst<"svandv[_{d}]", "sPd", "csilUcUsUiUl", MergeNone, "aarch64_sve_andv">;
875def SVEORV : SInst<"sveorv[_{d}]", "sPd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eorv">;
876def SVMAXV_S : SInst<"svmaxv[_{d}]", "sPd", "csil", MergeNone, "aarch64_sve_smaxv">;
877def SVMAXV_U : SInst<"svmaxv[_{d}]", "sPd", "UcUsUiUl", MergeNone, "aarch64_sve_umaxv">;
878def SVMINV_S : SInst<"svminv[_{d}]", "sPd", "csil", MergeNone, "aarch64_sve_sminv">;
879def SVMINV_U : SInst<"svminv[_{d}]", "sPd", "UcUsUiUl", MergeNone, "aarch64_sve_uminv">;
880def SVORV : SInst<"svorv[_{d}]", "sPd", "csilUcUsUiUl", MergeNone, "aarch64_sve_orv">;
881
882////////////////////////////////////////////////////////////////////////////////
883// Integer comparisons
884
885def SVCMPEQ : SInst<"svcmpeq[_{d}]", "PPdd", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpeq">;
886def SVCMPNE : SInst<"svcmpne[_{d}]", "PPdd", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpne">;
887def SVCMPGE : SInst<"svcmpge[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpge">;
888def SVCMPGT : SInst<"svcmpgt[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpgt">;
889def SVCMPLE : SInst<"svcmple[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpge", [ReverseCompare]>;
890def SVCMPLT : SInst<"svcmplt[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpgt", [ReverseCompare]>;
891def SVCMPHI : SInst<"svcmpgt[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi">;
892def SVCMPHS : SInst<"svcmpge[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs">;
893def SVCMPLO : SInst<"svcmplt[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi", [ReverseCompare]>;
894def SVCMPLS : SInst<"svcmple[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs", [ReverseCompare]>;
895
896def SVCMPEQ_N : SInst<"svcmpeq[_n_{d}]", "PPda", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpeq">;
897def SVCMPNE_N : SInst<"svcmpne[_n_{d}]", "PPda", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpne">;
898def SVCMPGE_N : SInst<"svcmpge[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpge">;
899def SVCMPGT_N : SInst<"svcmpgt[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpgt">;
900def SVCMPLE_N : SInst<"svcmple[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpge", [ReverseCompare]>;
901def SVCMPLT_N : SInst<"svcmplt[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpgt", [ReverseCompare]>;
902def SVCMPHS_N : SInst<"svcmpge[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs">;
903def SVCMPHI_N : SInst<"svcmpgt[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi">;
904def SVCMPLS_N : SInst<"svcmple[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs", [ReverseCompare]>;
905def SVCMPLO_N : SInst<"svcmplt[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi", [ReverseCompare]>;
906
907def SVCMPEQ_WIDE : SInst<"svcmpeq_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpeq_wide">;
908def SVCMPNE_WIDE : SInst<"svcmpne_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpne_wide">;
909def SVCMPGE_WIDE : SInst<"svcmpge_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpge_wide">;
910def SVCMPGT_WIDE : SInst<"svcmpgt_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpgt_wide">;
911def SVCMPLE_WIDE : SInst<"svcmple_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmple_wide">;
912def SVCMPLT_WIDE : SInst<"svcmplt_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmplt_wide">;
913def SVCMPHI_WIDE : SInst<"svcmpgt_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmphi_wide">;
914def SVCMPHS_WIDE : SInst<"svcmpge_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmphs_wide">;
915def SVCMPLO_WIDE : SInst<"svcmplt_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmplo_wide">;
916def SVCMPLS_WIDE : SInst<"svcmple_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmpls_wide">;
917
918def SVCMPEQ_WIDE_N : SInst<"svcmpeq_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpeq_wide">;
919def SVCMPNE_WIDE_N : SInst<"svcmpne_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpne_wide">;
920def SVCMPGE_WIDE_N : SInst<"svcmpge_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpge_wide">;
921def SVCMPGT_WIDE_N : SInst<"svcmpgt_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpgt_wide">;
922def SVCMPLE_WIDE_N : SInst<"svcmple_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmple_wide">;
923def SVCMPLT_WIDE_N : SInst<"svcmplt_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmplt_wide">;
924def SVCMPHS_WIDE_N : SInst<"svcmpge_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmphs_wide">;
925def SVCMPHI_WIDE_N : SInst<"svcmpgt_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmphi_wide">;
926def SVCMPLO_WIDE_N : SInst<"svcmplt_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmplo_wide">;
927def SVCMPLS_WIDE_N : SInst<"svcmple_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmpls_wide">;
928
929////////////////////////////////////////////////////////////////////////////////
930// While comparisons
931
932def SVWHILELE_S32 : SInst<"svwhilele_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilele", [IsOverloadWhile]>;
933def SVWHILELE_S64 : SInst<"svwhilele_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilele", [IsOverloadWhile]>;
934def SVWHILELO_U32 : SInst<"svwhilelt_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilelo", [IsOverloadWhile]>;
935def SVWHILELO_U64 : SInst<"svwhilelt_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilelo", [IsOverloadWhile]>;
936def SVWHILELS_U32 : SInst<"svwhilele_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilels", [IsOverloadWhile]>;
937def SVWHILELS_U64 : SInst<"svwhilele_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilels", [IsOverloadWhile]>;
938def SVWHILELT_S32 : SInst<"svwhilelt_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilelt", [IsOverloadWhile]>;
939def SVWHILELT_S64 : SInst<"svwhilelt_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilelt", [IsOverloadWhile]>;
940
941////////////////////////////////////////////////////////////////////////////////
942// Counting bit
943
944multiclass SInstCLS<string name, string types, string intrinsic, list<FlagType> flags=[]> {
945 def _M : SInst<name # "[_{d}]", "uuPd", types, MergeOp1, intrinsic, flags>;
946 def _X : SInst<name # "[_{d}]", "uPd", types, MergeAnyExp, intrinsic, flags>;
947 def _Z : SInst<name # "[_{d}]", "uPd", types, MergeZeroExp, intrinsic, flags>;
948}
949
950defm SVCLS : SInstCLS<"svcls", "csil", "aarch64_sve_cls">;
951defm SVCLZ : SInstCLS<"svclz", "csilUcUsUiUl", "aarch64_sve_clz">;
952defm SVCNT : SInstCLS<"svcnt", "csilUcUsUiUlhfd", "aarch64_sve_cnt">;
953
954let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
955 defm SVCNT_BF16 : SInstCLS<"svcnt", "b", "aarch64_sve_cnt">;
956}
957
958////////////////////////////////////////////////////////////////////////////////
959// Conversion
960
961defm SVEXTB_S : SInstZPZ<"svextb", "sil", "aarch64_sve_sxtb">;
962defm SVEXTB_U : SInstZPZ<"svextb", "UsUiUl", "aarch64_sve_uxtb">;
963defm SVEXTH_S : SInstZPZ<"svexth", "il", "aarch64_sve_sxth">;
964defm SVEXTH_U : SInstZPZ<"svexth", "UiUl", "aarch64_sve_uxth">;
965defm SVEXTW_S : SInstZPZ<"svextw", "l", "aarch64_sve_sxtw">;
966defm SVEXTW_U : SInstZPZ<"svextw", "Ul", "aarch64_sve_uxtw">;
967
968////////////////////////////////////////////////////////////////////////////////
969// Reversal
970
971defm SVRBIT : SInstZPZ<"svrbit", "csilUcUsUiUl", "aarch64_sve_rbit">;
972defm SVREVB : SInstZPZ<"svrevb", "silUsUiUl", "aarch64_sve_revb">;
973defm SVREVH : SInstZPZ<"svrevh", "ilUiUl", "aarch64_sve_revh">;
974defm SVREVW : SInstZPZ<"svrevw", "lUl", "aarch64_sve_revw">;
975
976////////////////////////////////////////////////////////////////////////////////
977// Floating-point arithmetic
978
979defm SVABS_F : SInstZPZ<"svabs", "hfd", "aarch64_sve_fabs">;
980defm SVNEG_F : SInstZPZ<"svneg", "hfd", "aarch64_sve_fneg">;
981
982defm SVABD_F : SInstZPZZ<"svabd", "hfd", "aarch64_sve_fabd">;
983defm SVADD_F : SInstZPZZ<"svadd", "hfd", "aarch64_sve_fadd">;
984defm SVDIV_F : SInstZPZZ<"svdiv", "hfd", "aarch64_sve_fdiv">;
985defm SVDIVR_F : SInstZPZZ<"svdivr", "hfd", "aarch64_sve_fdivr">;
986defm SVMAX_F : SInstZPZZ<"svmax", "hfd", "aarch64_sve_fmax">;
987defm SVMAXNM : SInstZPZZ<"svmaxnm","hfd", "aarch64_sve_fmaxnm">;
988defm SVMIN_F : SInstZPZZ<"svmin", "hfd", "aarch64_sve_fmin">;
989defm SVMINNM : SInstZPZZ<"svminnm","hfd", "aarch64_sve_fminnm">;
990defm SVMUL_F : SInstZPZZ<"svmul", "hfd", "aarch64_sve_fmul">;
991defm SVMULX : SInstZPZZ<"svmulx", "hfd", "aarch64_sve_fmulx">;
992defm SVSUB_F : SInstZPZZ<"svsub", "hfd", "aarch64_sve_fsub">;
993defm SVSUBR_F : SInstZPZZ<"svsubr", "hfd", "aarch64_sve_fsubr">;
994
995defm SVRECPX : SInstZPZ<"svrecpx", "hfd", "aarch64_sve_frecpx">;
996defm SVRINTA : SInstZPZ<"svrinta", "hfd", "aarch64_sve_frinta">;
997defm SVRINTI : SInstZPZ<"svrinti", "hfd", "aarch64_sve_frinti">;
998defm SVRINTM : SInstZPZ<"svrintm", "hfd", "aarch64_sve_frintm">;
999defm SVRINTN : SInstZPZ<"svrintn", "hfd", "aarch64_sve_frintn">;
1000defm SVRINTP : SInstZPZ<"svrintp", "hfd", "aarch64_sve_frintp">;
1001defm SVRINTX : SInstZPZ<"svrintx", "hfd", "aarch64_sve_frintx">;
1002defm SVRINTZ : SInstZPZ<"svrintz", "hfd", "aarch64_sve_frintz">;
1003defm SVSQRT : SInstZPZ<"svsqrt", "hfd", "aarch64_sve_fsqrt">;
1004
1005def SVEXPA : SInst<"svexpa[_{d}]", "du", "hfd", MergeNone, "aarch64_sve_fexpa_x">;
1006def SVTMAD : SInst<"svtmad[_{d}]", "dddi", "hfd", MergeNone, "aarch64_sve_ftmad_x", [], [ImmCheck<2, ImmCheck0_7>]>;
1007def SVTSMUL : SInst<"svtsmul[_{d}]", "ddu", "hfd", MergeNone, "aarch64_sve_ftsmul_x">;
1008def SVTSSEL : SInst<"svtssel[_{d}]", "ddu", "hfd", MergeNone, "aarch64_sve_ftssel_x">;
1009
1010def SVSCALE_M : SInst<"svscale[_{d}]", "dPdx", "hfd", MergeOp1, "aarch64_sve_fscale">;
1011def SVSCALE_X : SInst<"svscale[_{d}]", "dPdx", "hfd", MergeAny, "aarch64_sve_fscale">;
1012def SVSCALE_Z : SInst<"svscale[_{d}]", "dPdx", "hfd", MergeZero, "aarch64_sve_fscale">;
1013
1014def SVSCALE_N_M : SInst<"svscale[_n_{d}]", "dPdK", "hfd", MergeOp1, "aarch64_sve_fscale">;
1015def SVSCALE_N_X : SInst<"svscale[_n_{d}]", "dPdK", "hfd", MergeAny, "aarch64_sve_fscale">;
1016def SVSCALE_N_Z : SInst<"svscale[_n_{d}]", "dPdK", "hfd", MergeZero, "aarch64_sve_fscale">;
1017
1018defm SVMAD_F : SInstZPZZZ<"svmad", "hfd", "aarch64_sve_fmad">;
1019defm SVMLA_F : SInstZPZZZ<"svmla", "hfd", "aarch64_sve_fmla">;
1020defm SVMLS_F : SInstZPZZZ<"svmls", "hfd", "aarch64_sve_fmls">;
1021defm SVMSB_F : SInstZPZZZ<"svmsb", "hfd", "aarch64_sve_fmsb">;
1022defm SVNMAD_F : SInstZPZZZ<"svnmad", "hfd", "aarch64_sve_fnmad">;
1023defm SVNMLA_F : SInstZPZZZ<"svnmla", "hfd", "aarch64_sve_fnmla">;
1024defm SVNMLS_F : SInstZPZZZ<"svnmls", "hfd", "aarch64_sve_fnmls">;
1025defm SVNMSB_F : SInstZPZZZ<"svnmsb", "hfd", "aarch64_sve_fnmsb">;
1026
1027def SVCADD_M : SInst<"svcadd[_{d}]", "dPddi", "hfd", MergeOp1, "aarch64_sve_fcadd", [], [ImmCheck<3, ImmCheckComplexRot90_270>]>;
1028def SVCADD_X : SInst<"svcadd[_{d}]", "dPddi", "hfd", MergeAny, "aarch64_sve_fcadd", [], [ImmCheck<3, ImmCheckComplexRot90_270>]>;
1029def SVCADD_Z : SInst<"svcadd[_{d}]", "dPddi", "hfd", MergeZero, "aarch64_sve_fcadd", [], [ImmCheck<3, ImmCheckComplexRot90_270>]>;
1030def SVCMLA_M : SInst<"svcmla[_{d}]", "dPdddi", "hfd", MergeOp1, "aarch64_sve_fcmla", [], [ImmCheck<4, ImmCheckComplexRotAll90>]>;
1031def SVCMLA_X : SInst<"svcmla[_{d}]", "dPdddi", "hfd", MergeAny, "aarch64_sve_fcmla", [], [ImmCheck<4, ImmCheckComplexRotAll90>]>;
1032def SVCMLA_Z : SInst<"svcmla[_{d}]", "dPdddi", "hfd", MergeZero, "aarch64_sve_fcmla", [], [ImmCheck<4, ImmCheckComplexRotAll90>]>;
1033
1034def SVCMLA_LANE : SInst<"svcmla_lane[_{d}]", "ddddii", "hf", MergeNone, "aarch64_sve_fcmla_lane", [], [ImmCheck<3, ImmCheckLaneIndexCompRotate, 2>,
1035 ImmCheck<4, ImmCheckComplexRotAll90>]>;
1036def SVMLA_LANE : SInst<"svmla_lane[_{d}]", "ddddi", "hfd", MergeNone, "aarch64_sve_fmla_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>;
1037def SVMLS_LANE : SInst<"svmls_lane[_{d}]", "ddddi", "hfd", MergeNone, "aarch64_sve_fmls_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>;
1038def SVMUL_LANE : SInst<"svmul_lane[_{d}]", "dddi", "hfd", MergeNone, "aarch64_sve_fmul_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>;
1039
1040def SVRECPE : SInst<"svrecpe[_{d}]", "dd", "hfd", MergeNone, "aarch64_sve_frecpe_x">;
1041def SVRECPS : SInst<"svrecps[_{d}]", "ddd", "hfd", MergeNone, "aarch64_sve_frecps_x">;
1042def SVRSQRTE : SInst<"svrsqrte[_{d}]", "dd", "hfd", MergeNone, "aarch64_sve_frsqrte_x">;
1043def SVRSQRTS : SInst<"svrsqrts[_{d}]", "ddd", "hfd", MergeNone, "aarch64_sve_frsqrts_x">;
1044
1045////////////////////////////////////////////////////////////////////////////////
1046// Floating-point reductions
1047
1048def SVFADDA : SInst<"svadda[_{d}]", "sPsd", "hfd", MergeNone, "aarch64_sve_fadda">;
1049def SVFADDV : SInst<"svaddv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_faddv">;
1050def SVFMAXV : SInst<"svmaxv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fmaxv">;
1051def SVFMAXNMV : SInst<"svmaxnmv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fmaxnmv">;
1052def SVFMINV : SInst<"svminv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fminv">;
1053def SVFMINNMV : SInst<"svminnmv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fminnmv">;
1054
1055////////////////////////////////////////////////////////////////////////////////
1056// Floating-point comparisons
1057
1058def SVACGE : SInst<"svacge[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facge">;
1059def SVACGT : SInst<"svacgt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facgt">;
1060def SVACLE : SInst<"svacle[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facge", [ReverseCompare]>;
1061def SVACLT : SInst<"svaclt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facgt", [ReverseCompare]>;
1062def SVCMPUO : SInst<"svcmpuo[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpuo">;
1063
1064def SVACGE_N : SInst<"svacge[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_facge">;
1065def SVACGT_N : SInst<"svacgt[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_facgt">;
1066def SVACLE_N : SInst<"svacle[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_facge", [ReverseCompare]>;
1067def SVACLT_N : SInst<"svaclt[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_facgt", [ReverseCompare]>;
1068def SVCMPUO_N : SInst<"svcmpuo[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpuo">;
1069
1070def SVCMPEQ_F : SInst<"svcmpeq[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpeq">;
1071def SVCMPNE_F : SInst<"svcmpne[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpne">;
1072def SVCMPGE_F : SInst<"svcmpge[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpge">;
1073def SVCMPGT_F : SInst<"svcmpgt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpgt">;
1074def SVCMPLE_F : SInst<"svcmple[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpge", [ReverseCompare]>;
1075def SVCMPLT_F : SInst<"svcmplt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpgt", [ReverseCompare]>;
1076
1077def SVCMPEQ_F_N : SInst<"svcmpeq[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpeq">;
1078def SVCMPNE_F_N : SInst<"svcmpne[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpne">;
1079def SVCMPGE_F_N : SInst<"svcmpge[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpge">;
1080def SVCMPGT_F_N : SInst<"svcmpgt[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpgt">;
1081def SVCMPLE_F_N : SInst<"svcmple[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpge", [ReverseCompare]>;
1082def SVCMPLT_F_N : SInst<"svcmplt[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpgt", [ReverseCompare]>;
1083
1084////////////////////////////////////////////////////////////////////////////////
1085// Floating-point conversions
1086
1087multiclass SInstCvtMXZ<
1088 string name, string m_types, string xz_types, string types,
1089 string intrinsic, list<FlagType> flags = [IsOverloadNone]> {
1090 def _M : SInst<name, m_types, types, MergeOp1, intrinsic, flags>;
1091 def _X : SInst<name, xz_types, types, MergeAnyExp, intrinsic, flags>;
1092 def _Z : SInst<name, xz_types, types, MergeZeroExp, intrinsic, flags>;
1093}
1094
1095multiclass SInstCvtMX<string name, string m_types, string xz_types,
1096 string types, string intrinsic,
1097 list<FlagType> flags = [IsOverloadNone]> {
1098 def _M : SInst<name, m_types, types, MergeOp1, intrinsic, flags>;
1099 def _X : SInst<name, xz_types, types, MergeAnyExp, intrinsic, flags>;
1100}
1101
1102// svcvt_s##_f16
1103defm SVFCVTZS_S16_F16 : SInstCvtMXZ<"svcvt_s16[_f16]", "ddPO", "dPO", "s", "aarch64_sve_fcvtzs", [IsOverloadCvt]>;
1104defm SVFCVTZS_S32_F16 : SInstCvtMXZ<"svcvt_s32[_f16]", "ddPO", "dPO", "i", "aarch64_sve_fcvtzs_i32f16">;
1105defm SVFCVTZS_S64_F16 : SInstCvtMXZ<"svcvt_s64[_f16]", "ddPO", "dPO", "l", "aarch64_sve_fcvtzs_i64f16">;
1106
1107// svcvt_s##_f32
1108defm SVFCVTZS_S32_F32 : SInstCvtMXZ<"svcvt_s32[_f32]", "ddPM", "dPM", "i", "aarch64_sve_fcvtzs", [IsOverloadCvt]>;
1109defm SVFCVTZS_S64_F32 : SInstCvtMXZ<"svcvt_s64[_f32]", "ddPM", "dPM", "l", "aarch64_sve_fcvtzs_i64f32">;
1110
1111let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
1112 defm SVCVT_BF16_F32 : SInstCvtMXZ<"svcvt_bf16[_f32]", "ddPM", "dPM", "b", "aarch64_sve_fcvt_bf16f32">;
1113 def SVCVTNT_BF16_F32 : SInst<"svcvtnt_bf16[_f32]", "ddPM", "b", MergeOp1, "aarch64_sve_fcvtnt_bf16f32", [IsOverloadNone]>;
1114}
1115
1116// svcvt_s##_f64
1117defm SVFCVTZS_S32_F64 : SInstCvtMXZ<"svcvt_s32[_f64]", "ttPd", "tPd", "d", "aarch64_sve_fcvtzs_i32f64">;
1118defm SVFCVTZS_S64_F64 : SInstCvtMXZ<"svcvt_s64[_f64]", "ddPN", "dPN", "l", "aarch64_sve_fcvtzs", [IsOverloadCvt]>;
1119
1120// svcvt_u##_f16
1121defm SVFCVTZU_U16_F16 : SInstCvtMXZ<"svcvt_u16[_f16]", "ddPO", "dPO", "Us", "aarch64_sve_fcvtzu", [IsOverloadCvt]>;
1122defm SVFCVTZU_U32_F16 : SInstCvtMXZ<"svcvt_u32[_f16]", "ddPO", "dPO", "Ui", "aarch64_sve_fcvtzu_i32f16">;
1123defm SVFCVTZU_U64_F16 : SInstCvtMXZ<"svcvt_u64[_f16]", "ddPO", "dPO", "Ul", "aarch64_sve_fcvtzu_i64f16">;
1124
1125// svcvt_u##_f32
1126defm SVFCVTZU_U32_F32 : SInstCvtMXZ<"svcvt_u32[_f32]", "ddPM", "dPM", "Ui", "aarch64_sve_fcvtzu", [IsOverloadCvt]>;
1127defm SVFCVTZU_U64_F32 : SInstCvtMXZ<"svcvt_u64[_f32]", "ddPM", "dPM", "Ul", "aarch64_sve_fcvtzu_i64f32">;
1128
1129// svcvt_u##_f64
1130defm SVFCVTZU_U32_F64 : SInstCvtMXZ<"svcvt_u32[_f64]", "zzPd", "zPd", "d", "aarch64_sve_fcvtzu_i32f64">;
1131defm SVFCVTZU_U64_F64 : SInstCvtMXZ<"svcvt_u64[_f64]", "ddPN", "dPN", "Ul", "aarch64_sve_fcvtzu", [IsOverloadCvt]>;
1132
1133// svcvt_f16_s##
1134defm SVFCVTZS_F16_S16 : SInstCvtMXZ<"svcvt_f16[_s16]", "OOPd", "OPd", "s", "aarch64_sve_scvtf", [IsOverloadCvt]>;
1135defm SVFCVTZS_F16_S32 : SInstCvtMXZ<"svcvt_f16[_s32]", "OOPd", "OPd", "i", "aarch64_sve_scvtf_f16i32">;
1136defm SVFCVTZS_F16_S64 : SInstCvtMXZ<"svcvt_f16[_s64]", "OOPd", "OPd", "l", "aarch64_sve_scvtf_f16i64">;
1137
1138// svcvt_f32_s##
1139defm SVFCVTZS_F32_S32 : SInstCvtMXZ<"svcvt_f32[_s32]", "MMPd", "MPd", "i", "aarch64_sve_scvtf", [IsOverloadCvt]>;
1140defm SVFCVTZS_F32_S64 : SInstCvtMXZ<"svcvt_f32[_s64]", "MMPd", "MPd", "l", "aarch64_sve_scvtf_f32i64">;
1141
1142// svcvt_f64_s##
1143defm SVFCVTZS_F64_S32 : SInstCvtMXZ<"svcvt_f64[_s32]", "ddPt", "dPt", "d", "aarch64_sve_scvtf_f64i32">;
1144defm SVFCVTZS_F64_S64 : SInstCvtMXZ<"svcvt_f64[_s64]", "NNPd", "NPd", "l", "aarch64_sve_scvtf", [IsOverloadCvt]>;
1145
1146// svcvt_f16_u##
1147defm SVFCVTZU_F16_U16 : SInstCvtMXZ<"svcvt_f16[_u16]", "OOPd", "OPd", "Us", "aarch64_sve_ucvtf", [IsOverloadCvt]>;
1148defm SVFCVTZU_F16_U32 : SInstCvtMXZ<"svcvt_f16[_u32]", "OOPd", "OPd", "Ui", "aarch64_sve_ucvtf_f16i32">;
1149defm SVFCVTZU_F16_U64 : SInstCvtMXZ<"svcvt_f16[_u64]", "OOPd", "OPd", "Ul", "aarch64_sve_ucvtf_f16i64">;
1150
1151// svcvt_f32_u##
1152defm SVFCVTZU_F32_U32 : SInstCvtMXZ<"svcvt_f32[_u32]", "MMPd", "MPd", "Ui", "aarch64_sve_ucvtf", [IsOverloadCvt]>;
1153defm SVFCVTZU_F32_U64 : SInstCvtMXZ<"svcvt_f32[_u64]", "MMPd", "MPd", "Ul", "aarch64_sve_ucvtf_f32i64">;
1154
1155// svcvt_f64_u##
1156defm SVFCVTZU_F64_U32 : SInstCvtMXZ<"svcvt_f64[_u32]", "ddPz", "dPz", "d", "aarch64_sve_ucvtf_f64i32">;
1157defm SVFCVTZU_F64_U64 : SInstCvtMXZ<"svcvt_f64[_u64]", "NNPd", "NPd", "Ul", "aarch64_sve_ucvtf", [IsOverloadCvt]>;
1158
1159// svcvt_f16_f##
1160defm SVFCVT_F16_F32 : SInstCvtMXZ<"svcvt_f16[_f32]", "OOPd", "OPd", "f", "aarch64_sve_fcvt_f16f32">;
1161defm SVFCVT_F16_F64 : SInstCvtMXZ<"svcvt_f16[_f64]", "OOPd", "OPd", "d", "aarch64_sve_fcvt_f16f64">;
1162
1163// svcvt_f32_f##
1164defm SVFCVT_F32_F16 : SInstCvtMXZ<"svcvt_f32[_f16]", "ddPO", "dPO", "f", "aarch64_sve_fcvt_f32f16">;
1165defm SVFCVT_F32_F64 : SInstCvtMXZ<"svcvt_f32[_f64]", "MMPd", "MPd", "d", "aarch64_sve_fcvt_f32f64">;
1166
1167// svcvt_f64_f##
1168defm SVFCVT_F64_F16 : SInstCvtMXZ<"svcvt_f64[_f16]", "ddPO", "dPO", "d", "aarch64_sve_fcvt_f64f16">;
1169defm SVFCVT_F64_F32 : SInstCvtMXZ<"svcvt_f64[_f32]", "ddPM", "dPM", "d", "aarch64_sve_fcvt_f64f32">;
1170
1171let ArchGuard = "defined(__ARM_FEATURE_SVE2)" in {
1172defm SVCVTLT_F32 : SInstCvtMX<"svcvtlt_f32[_f16]", "ddPh", "dPh", "f", "aarch64_sve_fcvtlt_f32f16">;
1173defm SVCVTLT_F64 : SInstCvtMX<"svcvtlt_f64[_f32]", "ddPh", "dPh", "d", "aarch64_sve_fcvtlt_f64f32">;
1174
1175defm SVCVTX_F32 : SInstCvtMXZ<"svcvtx_f32[_f64]", "MMPd", "MPd", "d", "aarch64_sve_fcvtx_f32f64">;
1176
1177def SVCVTNT_F32 : SInst<"svcvtnt_f16[_f32]", "hhPd", "f", MergeOp1, "aarch64_sve_fcvtnt_f16f32", [IsOverloadNone]>;
1178def SVCVTNT_F64 : SInst<"svcvtnt_f32[_f64]", "hhPd", "d", MergeOp1, "aarch64_sve_fcvtnt_f32f64", [IsOverloadNone]>;
1179// SVCVTNT_X : Implemented as macro by SveEmitter.cpp
1180
1181def SVCVTXNT_F32 : SInst<"svcvtxnt_f32[_f64]", "MMPd", "d", MergeOp1, "aarch64_sve_fcvtxnt_f32f64", [IsOverloadNone]>;
1182// SVCVTXNT_X_F32 : Implemented as macro by SveEmitter.cpp
1183
1184}
1185
1186////////////////////////////////////////////////////////////////////////////////
1187// Permutations and selection
1188
1189multiclass SVEPerm<string name, string proto, string i> {
1190 def : SInst<name, proto, "csilUcUsUiUlhfd", MergeNone, i>;
1191 let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
1192 def: SInst<name, proto, "b", MergeNone, i>;
1193 }
1194}
1195
1196defm SVCLASTA : SVEPerm<"svclasta[_{d}]", "dPdd", "aarch64_sve_clasta">;
1197defm SVCLASTA_N : SVEPerm<"svclasta[_n_{d}]", "sPsd", "aarch64_sve_clasta_n">;
1198defm SVCLASTB : SVEPerm<"svclastb[_{d}]", "dPdd", "aarch64_sve_clastb">;
1199defm SVCLASTB_N : SVEPerm<"svclastb[_n_{d}]", "sPsd", "aarch64_sve_clastb_n">;
1200
1201def SVCOMPACT : SInst<"svcompact[_{d}]", "dPd", "ilUiUlfd", MergeNone, "aarch64_sve_compact">;
1202// Note: svdup_lane is implemented using the intrinsic for TBL to represent a
1203// splat of any possible lane. It is upto LLVM to pick a more efficient
1204// instruction such as DUP (indexed) if the lane index fits the range of the
1205// instruction's immediate.
1206def SVDUP_LANE : SInst<"svdup_lane[_{d}]", "ddL", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tbl">;
1207let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
1208def SVDUP_LANE_BF16 :
1209 SInst<"svdup_lane[_{d}]", "ddL", "b", MergeNone, "aarch64_sve_tbl">;
1210}
1211
1212def SVDUPQ_LANE : SInst<"svdupq_lane[_{d}]", "ddn", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_dupq_lane">;
1213let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
1214 def SVDUPQ_LANE_BF16 : SInst<"svdupq_lane[_{d}]", "ddn", "b", MergeNone, "aarch64_sve_dupq_lane">;
1215}
1216def SVEXT : SInst<"svext[_{d}]", "dddi", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ext", [], [ImmCheck<2, ImmCheckExtract, 1>]>;
1217defm SVLASTA : SVEPerm<"svlasta[_{d}]", "sPd", "aarch64_sve_lasta">;
1218defm SVLASTB : SVEPerm<"svlastb[_{d}]", "sPd", "aarch64_sve_lastb">;
1219def SVREV : SInst<"svrev[_{d}]", "dd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_rev">;
1220def SVSEL : SInst<"svsel[_{d}]", "dPdd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_sel">;
1221def SVSPLICE : SInst<"svsplice[_{d}]", "dPdd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_splice">;
1222def SVTBL : SInst<"svtbl[_{d}]", "ddu", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tbl">;
1223
1224let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
1225 def SVTBL_BF16 : SInst<"svtbl[_{d}]", "ddu", "b", MergeNone, "aarch64_sve_tbl">;
1226}
1227
1228def SVTRN1 : SInst<"svtrn1[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn1">;
1229def SVTRN2 : SInst<"svtrn2[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn2">;
1230def SVUNPKHI_S : SInst<"svunpkhi[_{d}]", "dh", "sil", MergeNone, "aarch64_sve_sunpkhi">;
1231def SVUNPKHI_U : SInst<"svunpkhi[_{d}]", "dh", "UsUiUl", MergeNone, "aarch64_sve_uunpkhi">;
1232def SVUNPKLO_S : SInst<"svunpklo[_{d}]", "dh", "sil", MergeNone, "aarch64_sve_sunpklo">;
1233def SVUNPKLO_U : SInst<"svunpklo[_{d}]", "dh", "UsUiUl", MergeNone, "aarch64_sve_uunpklo">;
1234def SVUZP1 : SInst<"svuzp1[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp1">;
1235def SVUZP2 : SInst<"svuzp2[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp2">;
1236def SVZIP1 : SInst<"svzip1[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip1">;
1237def SVZIP2 : SInst<"svzip2[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip2">;
1238
1239let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
1240def SVEXT_BF16 : SInst<"svext[_{d}]", "dddi", "b", MergeNone, "aarch64_sve_ext", [], [ImmCheck<2, ImmCheckExtract, 1>]>;
1241def SVREV_BF16 : SInst<"svrev[_{d}]", "dd", "b", MergeNone, "aarch64_sve_rev">;
1242def SVSEL_BF16 : SInst<"svsel[_{d}]", "dPdd", "b", MergeNone, "aarch64_sve_sel">;
1243def SVSPLICE_BF16 : SInst<"svsplice[_{d}]", "dPdd", "b", MergeNone, "aarch64_sve_splice">;
1244def SVTRN1_BF16 : SInst<"svtrn1[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_trn1">;
1245def SVTRN2_BF16 : SInst<"svtrn2[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_trn2">;
1246def SVUZP1_BF16 : SInst<"svuzp1[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_uzp1">;
1247def SVUZP2_BF16 : SInst<"svuzp2[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_uzp2">;
1248def SVZIP1_BF16 : SInst<"svzip1[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_zip1">;
1249def SVZIP2_BF16 : SInst<"svzip2[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_zip2">;
1250}
1251
1252def SVREV_B : SInst<"svrev_{d}", "PP", "PcPsPiPl", MergeNone, "aarch64_sve_rev">;
1253def SVSEL_B : SInst<"svsel[_b]", "PPPP", "Pc", MergeNone, "aarch64_sve_sel">;
1254def SVTRN1_B : SInst<"svtrn1_{d}", "PPP", "PcPsPiPl", MergeNone, "aarch64_sve_trn1">;
1255def SVTRN2_B : SInst<"svtrn2_{d}", "PPP", "PcPsPiPl", MergeNone, "aarch64_sve_trn2">;
1256def SVPUNPKHI : SInst<"svunpkhi[_b]", "PP", "Pc", MergeNone, "aarch64_sve_punpkhi">;
1257def SVPUNPKLO : SInst<"svunpklo[_b]", "PP", "Pc", MergeNone, "aarch64_sve_punpklo">;
1258def SVUZP1_B : SInst<"svuzp1_{d}", "PPP", "PcPsPiPl", MergeNone, "aarch64_sve_uzp1">;
1259def SVUZP2_B : SInst<"svuzp2_{d}", "PPP", "PcPsPiPl", MergeNone, "aarch64_sve_uzp2">;
1260def SVZIP1_B : SInst<"svzip1_{d}", "PPP", "PcPsPiPl", MergeNone, "aarch64_sve_zip1">;
1261def SVZIP2_B : SInst<"svzip2_{d}", "PPP", "PcPsPiPl", MergeNone, "aarch64_sve_zip2">;
1262
1263////////////////////////////////////////////////////////////////////////////////
1264// Predicate creation
1265
1266def SVPFALSE : SInst<"svpfalse[_b]", "P", "", MergeNone, "", [IsOverloadNone]>;
1267
1268def SVPTRUE_PAT : SInst<"svptrue_pat_{d}", "PI", "PcPsPiPl", MergeNone, "aarch64_sve_ptrue">;
1269def SVPTRUE : SInst<"svptrue_{d}", "P", "PcPsPiPl", MergeNone, "aarch64_sve_ptrue", [IsAppendSVALL]>;
1270
1271def SVDUPQ_B8 : SInst<"svdupq[_n]_{d}", "Pssssssssssssssss", "Pc", MergeNone>;
1272def SVDUPQ_B16 : SInst<"svdupq[_n]_{d}", "Pssssssss", "Ps", MergeNone>;
1273def SVDUPQ_B32 : SInst<"svdupq[_n]_{d}", "Pssss", "Pi", MergeNone>;
1274def SVDUPQ_B64 : SInst<"svdupq[_n]_{d}", "Pss", "Pl", MergeNone>;
1275def SVDUP_N_B : SInst<"svdup[_n]_{d}", "Ps", "PcPsPiPl", MergeNone>;
1276
1277
1278////////////////////////////////////////////////////////////////////////////////
1279// Predicate operations
1280
1281def SVAND_B_Z : SInst<"svand[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_and_z">;
1282def SVBIC_B_Z : SInst<"svbic[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_bic_z">;
1283def SVEOR_B_Z : SInst<"sveor[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_eor_z">;
1284def SVMOV_B_Z : SInst<"svmov[_b]_z", "PPP", "Pc", MergeNone>; // Uses custom expansion
1285def SVNAND_B_Z : SInst<"svnand[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_nand_z">;
1286def SVNOR_B_Z : SInst<"svnor[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_nor_z">;
1287def SVNOT_B_Z : SInst<"svnot[_b]_z", "PPP", "Pc", MergeNone>; // Uses custom expansion
1288def SVORN_B_Z : SInst<"svorn[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_orn_z">;
1289def SVORR_B_Z : SInst<"svorr[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_orr_z">;
1290
1291def SVBRKA : SInst<"svbrka[_b]_m", "PPPP", "Pc", MergeNone, "aarch64_sve_brka">;
1292def SVBRKA_Z : SInst<"svbrka[_b]_z", "PPP", "Pc", MergeNone, "aarch64_sve_brka_z">;
1293def SVBRKB : SInst<"svbrkb[_b]_m", "PPPP", "Pc", MergeNone, "aarch64_sve_brkb">;
1294def SVBRKB_Z : SInst<"svbrkb[_b]_z", "PPP", "Pc", MergeNone, "aarch64_sve_brkb_z">;
1295def SVBRKN_Z : SInst<"svbrkn[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_brkn_z">;
1296def SVBRKPA_Z : SInst<"svbrkpa[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_brkpa_z">;
1297def SVBRKPB_Z : SInst<"svbrkpb[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_brkpb_z">;
1298
1299def SVPFIRST : SInst<"svpfirst[_b]", "PPP", "Pc", MergeNone, "aarch64_sve_pfirst">;
1300def SVPNEXT : SInst<"svpnext_{d}", "PPP", "PcPsPiPl", MergeNone, "aarch64_sve_pnext">;
1301
1302////////////////////////////////////////////////////////////////////////////////
1303// Testing predicates
1304
1305def SVPTEST_ANY : SInst<"svptest_any", "sPP", "Pc", MergeNone, "aarch64_sve_ptest_any">;
1306def SVPTEST_FIRST : SInst<"svptest_first", "sPP", "Pc", MergeNone, "aarch64_sve_ptest_first">;
1307def SVPTEST_LAST : SInst<"svptest_last", "sPP", "Pc", MergeNone, "aarch64_sve_ptest_last">;
1308
1309////////////////////////////////////////////////////////////////////////////////
1310// FFR manipulation
1311
1312def SVRDFFR : SInst<"svrdffr", "P", "Pc", MergeNone, "", [IsOverloadNone]>;
1313def SVRDFFR_Z : SInst<"svrdffr_z", "PP", "Pc", MergeNone, "", [IsOverloadNone]>;
1314def SVSETFFR : SInst<"svsetffr", "v", "", MergeNone, "", [IsOverloadNone]>;
1315def SVWRFFR : SInst<"svwrffr", "vP", "Pc", MergeNone, "", [IsOverloadNone]>;
1316
1317////////////////////////////////////////////////////////////////////////////////
1318// Counting elements
1319
1320def SVCNTB_PAT : SInst<"svcntb_pat", "nI", "", MergeNone, "aarch64_sve_cntb", [IsOverloadNone]>;
1321def SVCNTH_PAT : SInst<"svcnth_pat", "nI", "", MergeNone, "aarch64_sve_cnth", [IsOverloadNone]>;
1322def SVCNTW_PAT : SInst<"svcntw_pat", "nI", "", MergeNone, "aarch64_sve_cntw", [IsOverloadNone]>;
1323def SVCNTD_PAT : SInst<"svcntd_pat", "nI", "", MergeNone, "aarch64_sve_cntd", [IsOverloadNone]>;
1324
1325def SVCNTB : SInst<"svcntb", "n", "", MergeNone, "aarch64_sve_cntb", [IsAppendSVALL, IsOverloadNone]>;
1326def SVCNTH : SInst<"svcnth", "n", "", MergeNone, "aarch64_sve_cnth", [IsAppendSVALL, IsOverloadNone]>;
1327def SVCNTW : SInst<"svcntw", "n", "", MergeNone, "aarch64_sve_cntw", [IsAppendSVALL, IsOverloadNone]>;
1328def SVCNTD : SInst<"svcntd", "n", "", MergeNone, "aarch64_sve_cntd", [IsAppendSVALL, IsOverloadNone]>;
1329
1330def SVCNTP : SInst<"svcntp_{d}", "nPP", "PcPsPiPl", MergeNone, "aarch64_sve_cntp">;
1331def SVLEN : SInst<"svlen[_{d}]", "nd", "csilUcUsUiUlhfd", MergeNone>;
1332
1333let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
1334def SVLEN_BF16 : SInst<"svlen[_{d}]", "nd", "b", MergeNone>;
1335}
1336
1337////////////////////////////////////////////////////////////////////////////////
1338// Saturating scalar arithmetic
1339
1340class sat_type<string u, string t> { string U = u; string T = t; }
1341def SignedByte : sat_type<"", "c">;
1342def SignedHalf : sat_type<"", "s">;
1343def SignedWord : sat_type<"", "i">;
1344def SignedDoubleWord : sat_type<"", "l">;
1345def UnsignedByte : sat_type<"U", "Uc">;
1346def UnsignedHalf : sat_type<"U", "Us">;
1347def UnsignedWord : sat_type<"U", "Ui">;
1348def UnsignedDoubleWord : sat_type<"U", "Ul">;
1349
1350multiclass SInst_SAT1<string name, string intrinsic, sat_type type> {
1351 def _N32 : SInst<name # "_pat[_n_{d}]", "ssIi", type.U # "i", MergeNone, intrinsic # "_n32", [IsOverloadNone], [ImmCheck<2, ImmCheck1_16>]>;
1352 def _N64 : SInst<name # "_pat[_n_{d}]", "ssIi", type.U # "l", MergeNone, intrinsic # "_n64", [IsOverloadNone], [ImmCheck<2, ImmCheck1_16>]>;
1353 def _N32_ALL : SInst<name # "[_n_{d}]", "ssi", type.U # "i", MergeNone, intrinsic # "_n32", [IsOverloadNone, IsInsertOp1SVALL], [ImmCheck<1, ImmCheck1_16>]>;
1354 def _N64_ALL : SInst<name # "[_n_{d}]", "ssi", type.U # "l", MergeNone, intrinsic # "_n64", [IsOverloadNone, IsInsertOp1SVALL], [ImmCheck<1, ImmCheck1_16>]>;
1355}
1356
1357multiclass SInst_SAT2<string name, string intrinsic, sat_type type> {
1358 def "" : SInst<name # "_pat[_{d}]", "ddIi", type.T, MergeNone, intrinsic, [], [ImmCheck<2, ImmCheck1_16>]>;
1359 def _ALL : SInst<name # "[_{d}]", "ddi", type.T, MergeNone, intrinsic, [IsInsertOp1SVALL], [ImmCheck<1, ImmCheck1_16>]>;
1360
1361 def _N32 : SInst<name # "_pat[_n_{d}]", "ssIi", type.U # "i", MergeNone, intrinsic # "_n32", [IsOverloadNone], [ImmCheck<2, ImmCheck1_16>]>;
1362 def _N64 : SInst<name # "_pat[_n_{d}]", "ssIi", type.U # "l", MergeNone, intrinsic # "_n64", [IsOverloadNone], [ImmCheck<2, ImmCheck1_16>]>;
1363 def _N32_ALL : SInst<name # "[_n_{d}]", "ssi", type.U # "i", MergeNone, intrinsic # "_n32", [IsOverloadNone, IsInsertOp1SVALL], [ImmCheck<1, ImmCheck1_16>]>;
1364 def _N64_ALL : SInst<name # "[_n_{d}]", "ssi", type.U # "l", MergeNone, intrinsic # "_n64", [IsOverloadNone, IsInsertOp1SVALL], [ImmCheck<1, ImmCheck1_16>]>;
1365}
1366
1367defm SVQDECB_S : SInst_SAT1<"svqdecb", "aarch64_sve_sqdecb", SignedByte>;
1368defm SVQDECB_U : SInst_SAT1<"svqdecb", "aarch64_sve_uqdecb", UnsignedByte>;
1369defm SVQDECH_S : SInst_SAT2<"svqdech", "aarch64_sve_sqdech", SignedHalf>;
1370defm SVQDECH_U : SInst_SAT2<"svqdech", "aarch64_sve_uqdech", UnsignedHalf>;
1371defm SVQDECW_S : SInst_SAT2<"svqdecw", "aarch64_sve_sqdecw", SignedWord>;
1372defm SVQDECW_U : SInst_SAT2<"svqdecw", "aarch64_sve_uqdecw", UnsignedWord>;
1373defm SVQDECD_S : SInst_SAT2<"svqdecd", "aarch64_sve_sqdecd", SignedDoubleWord>;
1374defm SVQDECD_U : SInst_SAT2<"svqdecd", "aarch64_sve_uqdecd", UnsignedDoubleWord>;
1375
1376defm SVQINCB_S : SInst_SAT1<"svqincb", "aarch64_sve_sqincb", SignedByte>;
1377defm SVQINCB_U : SInst_SAT1<"svqincb", "aarch64_sve_uqincb", UnsignedByte>;
1378defm SVQINCH_S : SInst_SAT2<"svqinch", "aarch64_sve_sqinch", SignedHalf>;
1379defm SVQINCH_U : SInst_SAT2<"svqinch", "aarch64_sve_uqinch", UnsignedHalf>;
1380defm SVQINCW_S : SInst_SAT2<"svqincw", "aarch64_sve_sqincw", SignedWord>;
1381defm SVQINCW_U : SInst_SAT2<"svqincw", "aarch64_sve_uqincw", UnsignedWord>;
1382defm SVQINCD_S : SInst_SAT2<"svqincd", "aarch64_sve_sqincd", SignedDoubleWord>;
1383defm SVQINCD_U : SInst_SAT2<"svqincd", "aarch64_sve_uqincd", UnsignedDoubleWord>;
1384
1385def SVQDECP_S : SInst<"svqdecp[_{d}]", "ddP", "sil", MergeNone, "aarch64_sve_sqdecp">;
1386def SVQDECP_U : SInst<"svqdecp[_{d}]", "ddP", "UsUiUl", MergeNone, "aarch64_sve_uqdecp">;
1387def SVQINCP_S : SInst<"svqincp[_{d}]", "ddP", "sil", MergeNone, "aarch64_sve_sqincp">;
1388def SVQINCP_U : SInst<"svqincp[_{d}]", "ddP", "UsUiUl", MergeNone, "aarch64_sve_uqincp">;
1389
1390def SVQDECP_N_S32 : SInst<"svqdecp[_n_s32]_{d}", "kkP", "PcPsPiPl", MergeNone, "aarch64_sve_sqdecp_n32">;
1391def SVQDECP_N_S64 : SInst<"svqdecp[_n_s64]_{d}", "llP", "PcPsPiPl", MergeNone, "aarch64_sve_sqdecp_n64">;
1392def SVQDECP_N_U32 : SInst<"svqdecp[_n_u32]_{d}", "mmP", "PcPsPiPl", MergeNone, "aarch64_sve_uqdecp_n32">;
1393def SVQDECP_N_U64 : SInst<"svqdecp[_n_u64]_{d}", "nnP", "PcPsPiPl", MergeNone, "aarch64_sve_uqdecp_n64">;
1394def SVQINCP_N_S32 : SInst<"svqincp[_n_s32]_{d}", "kkP", "PcPsPiPl", MergeNone, "aarch64_sve_sqincp_n32">;
1395def SVQINCP_N_S64 : SInst<"svqincp[_n_s64]_{d}", "llP", "PcPsPiPl", MergeNone, "aarch64_sve_sqincp_n64">;
1396def SVQINCP_N_U32 : SInst<"svqincp[_n_u32]_{d}", "mmP", "PcPsPiPl", MergeNone, "aarch64_sve_uqincp_n32">;
1397def SVQINCP_N_U64 : SInst<"svqincp[_n_u64]_{d}", "nnP", "PcPsPiPl", MergeNone, "aarch64_sve_uqincp_n64">;
1398
1399let ArchGuard = "defined(__ARM_FEATURE_SVE_MATMUL_INT8)" in {
1400def SVMLLA_S32 : SInst<"svmmla[_s32]", "ddqq","i", MergeNone, "aarch64_sve_smmla">;
1401def SVMLLA_U32 : SInst<"svmmla[_u32]", "ddqq","Ui", MergeNone, "aarch64_sve_ummla">;
1402def SVUSMLLA_S32 : SInst<"svusmmla[_s32]", "ddbq","i", MergeNone, "aarch64_sve_usmmla">;
1403
1404def SVUSDOT_S : SInst<"svusdot[_s32]", "ddbq", "i", MergeNone, "aarch64_sve_usdot">;
1405def SVUSDOT_N_S : SInst<"svusdot[_n_s32]", "ddbr", "i", MergeNone, "aarch64_sve_usdot">;
1406def SVSUDOT_S : SInst<"svsudot[_s32]", "ddqb", "i", MergeNone, "aarch64_sve_usdot", [ReverseUSDOT]>;
1407def SVSUDOT_N_S : SInst<"svsudot[_n_s32]", "ddq@", "i", MergeNone, "aarch64_sve_usdot", [ReverseUSDOT]>;
1408
1409def SVUSDOT_LANE_S : SInst<"svusdot_lane[_s32]", "ddbqi", "i", MergeNone, "aarch64_sve_usdot_lane", [], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>;
1410def SVSUDOT_LANE_S : SInst<"svsudot_lane[_s32]", "ddqbi", "i", MergeNone, "aarch64_sve_sudot_lane", [], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>;
1411}
1412
1413let ArchGuard = "defined(__ARM_FEATURE_SVE_MATMUL_FP32)" in {
1414def SVMLLA_F32 : SInst<"svmmla[_f32]", "dddd","f", MergeNone, "aarch64_sve_fmmla">;
1415}
1416
1417let ArchGuard = "defined(__ARM_FEATURE_SVE_MATMUL_FP64)" in {
1418def SVMLLA_F64 : SInst<"svmmla[_f64]", "dddd","d", MergeNone, "aarch64_sve_fmmla">;
1419def SVTRN1Q : SInst<"svtrn1q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn1q">;
1420def SVTRN2Q : SInst<"svtrn2q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn2q">;
1421def SVUZP1Q : SInst<"svuzp1q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp1q">;
1422def SVUZP2Q : SInst<"svuzp2q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp2q">;
1423def SVZIP1Q : SInst<"svzip1q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip1q">;
1424def SVZIP2Q : SInst<"svzip2q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip2q">;
1425}
1426
1427let ArchGuard = "defined(__ARM_FEATURE_SVE_MATMUL_FP64) && defined(__ARM_FEATURE_SVE_BF16)" in {
1428def SVTRN1Q_BF16 : SInst<"svtrn1q[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_trn1q">;
1429def SVTRN2Q_BF16 : SInst<"svtrn2q[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_trn2q">;
1430def SVUZP1Q_BF16 : SInst<"svuzp1q[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_uzp1q">;
1431def SVUZP2Q_BF16 : SInst<"svuzp2q[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_uzp2q">;
1432def SVZIP1Q_BF16 : SInst<"svzip1q[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_zip1q">;
1433def SVZIP2Q_BF16 : SInst<"svzip2q[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_zip2q">;
1434}
1435
1436////////////////////////////////////////////////////////////////////////////////
1437// Vector creation
1438def SVUNDEF_1 : SInst<"svundef_{d}", "d", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef]>;
1439def SVUNDEF_2 : SInst<"svundef2_{d}", "2", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef]>;
1440def SVUNDEF_3 : SInst<"svundef3_{d}", "3", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef]>;
1441def SVUNDEF_4 : SInst<"svundef4_{d}", "4", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef]>;
1442
1443def SVCREATE_2 : SInst<"svcreate2[_{d}]", "2dd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tuple_create2", [IsTupleCreate]>;
1444def SVCREATE_3 : SInst<"svcreate3[_{d}]", "3ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tuple_create3", [IsTupleCreate]>;
1445def SVCREATE_4 : SInst<"svcreate4[_{d}]", "4dddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tuple_create4", [IsTupleCreate]>;
1446
1447let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
1448def SVUNDEF_1_BF16 : SInst<"svundef_{d}", "d", "b", MergeNone, "", [IsUndef]>;
1449def SVUNDEF_2_BF16 : SInst<"svundef2_{d}", "2", "b", MergeNone, "", [IsUndef]>;
1450def SVUNDEF_3_BF16 : SInst<"svundef3_{d}", "3", "b", MergeNone, "", [IsUndef]>;
1451def SVUNDEF_4_BF16 : SInst<"svundef4_{d}", "4", "b", MergeNone, "", [IsUndef]>;
1452
1453def SVCREATE_2_BF16 : SInst<"svcreate2[_{d}]", "2dd", "b", MergeNone, "aarch64_sve_tuple_create2", [IsTupleCreate]>;
1454def SVCREATE_3_BF16 : SInst<"svcreate3[_{d}]", "3ddd", "b", MergeNone, "aarch64_sve_tuple_create3", [IsTupleCreate]>;
1455def SVCREATE_4_BF16 : SInst<"svcreate4[_{d}]", "4dddd", "b", MergeNone, "aarch64_sve_tuple_create4", [IsTupleCreate]>;
1456}
1457
1458////////////////////////////////////////////////////////////////////////////////
1459// Vector insertion and extraction
1460def SVGET_2 : SInst<"svget2[_{d}]", "d2i", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tuple_get", [IsTupleGet], [ImmCheck<1, ImmCheck0_1>]>;
1461def SVGET_3 : SInst<"svget3[_{d}]", "d3i", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tuple_get", [IsTupleGet], [ImmCheck<1, ImmCheck0_2>]>;
1462def SVGET_4 : SInst<"svget4[_{d}]", "d4i", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tuple_get", [IsTupleGet], [ImmCheck<1, ImmCheck0_3>]>;
1463
1464def SVSET_2 : SInst<"svset2[_{d}]", "22id", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tuple_set", [IsTupleSet], [ImmCheck<1, ImmCheck0_1>]>;
1465def SVSET_3 : SInst<"svset3[_{d}]", "33id", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tuple_set", [IsTupleSet], [ImmCheck<1, ImmCheck0_2>]>;
1466def SVSET_4 : SInst<"svset4[_{d}]", "44id", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tuple_set", [IsTupleSet], [ImmCheck<1, ImmCheck0_3>]>;
1467
1468let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in {
1469def SVGET_2_BF16 : SInst<"svget2[_{d}]", "d2i", "b", MergeNone, "aarch64_sve_tuple_get", [IsTupleGet], [ImmCheck<1, ImmCheck0_1>]>;
1470def SVGET_3_BF16 : SInst<"svget3[_{d}]", "d3i", "b", MergeNone, "aarch64_sve_tuple_get", [IsTupleGet], [ImmCheck<1, ImmCheck0_2>]>;
1471def SVGET_4_BF16 : SInst<"svget4[_{d}]", "d4i", "b", MergeNone, "aarch64_sve_tuple_get", [IsTupleGet], [ImmCheck<1, ImmCheck0_3>]>;
1472
1473def SVSET_2_BF16 : SInst<"svset2[_{d}]", "22id", "b", MergeNone, "aarch64_sve_tuple_set", [IsTupleSet], [ImmCheck<1, ImmCheck0_1>]>;
1474def SVSET_3_BF16 : SInst<"svset3[_{d}]", "33id", "b", MergeNone, "aarch64_sve_tuple_set", [IsTupleSet], [ImmCheck<1, ImmCheck0_2>]>;
1475def SVSET_4_BF16 : SInst<"svset4[_{d}]", "44id", "b", MergeNone, "aarch64_sve_tuple_set", [IsTupleSet], [ImmCheck<1, ImmCheck0_3>]>;
1476}
1477
1478////////////////////////////////////////////////////////////////////////////////
1479// SVE2 WhileGE/GT
1480let ArchGuard = "defined(__ARM_FEATURE_SVE2)" in {
1481def SVWHILEGE_S32 : SInst<"svwhilege_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilege", [IsOverloadWhile]>;
1482def SVWHILEGE_S64 : SInst<"svwhilege_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilege", [IsOverloadWhile]>;
1483def SVWHILEGT_S32 : SInst<"svwhilegt_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilegt", [IsOverloadWhile]>;
1484def SVWHILEGT_S64 : SInst<"svwhilegt_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilegt", [IsOverloadWhile]>;
1485def SVWHILEHI_U32 : SInst<"svwhilegt_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehi", [IsOverloadWhile]>;
1486def SVWHILEHI_U64 : SInst<"svwhilegt_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehi", [IsOverloadWhile]>;
1487def SVWHILEHS_U32 : SInst<"svwhilege_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhile]>;
1488def SVWHILEHS_U64 : SInst<"svwhilege_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhile]>;
1489}
1490
1491////////////////////////////////////////////////////////////////////////////////
1492// SVE2 - Uniform DSP operations
1493
1494let ArchGuard = "defined(__ARM_FEATURE_SVE2)" in {
1495defm SVQADD_S : SInstZPZZ<"svqadd", "csli", "aarch64_sve_sqadd">;
1496defm SVQADD_U : SInstZPZZ<"svqadd", "UcUsUiUl", "aarch64_sve_uqadd">;
1497defm SVHADD_S : SInstZPZZ<"svhadd", "csli", "aarch64_sve_shadd">;
1498defm SVHADD_U : SInstZPZZ<"svhadd", "UcUsUiUl", "aarch64_sve_uhadd">;
1499defm SVRHADD_S : SInstZPZZ<"svrhadd", "csli", "aarch64_sve_srhadd">;
1500defm SVRHADD_U : SInstZPZZ<"svrhadd", "UcUsUiUl", "aarch64_sve_urhadd">;
1501
1502defm SVQSUB_S : SInstZPZZ<"svqsub", "csli", "aarch64_sve_sqsub">;
1503defm SVQSUB_U : SInstZPZZ<"svqsub", "UcUsUiUl", "aarch64_sve_uqsub">;
1504defm SVQSUBR_S : SInstZPZZ<"svqsubr", "csli", "aarch64_sve_sqsubr">;
1505defm SVQSUBR_U : SInstZPZZ<"svqsubr", "UcUsUiUl", "aarch64_sve_uqsubr">;
1506defm SVHSUB_S : SInstZPZZ<"svhsub", "csli", "aarch64_sve_shsub">;
1507defm SVHSUB_U : SInstZPZZ<"svhsub", "UcUsUiUl", "aarch64_sve_uhsub">;
1508defm SVHSUBR_S : SInstZPZZ<"svhsubr", "csli", "aarch64_sve_shsubr">;
1509defm SVHSUBR_U : SInstZPZZ<"svhsubr", "UcUsUiUl", "aarch64_sve_uhsubr">;
1510
1511defm SVQABS : SInstZPZ<"svqabs", "csil", "aarch64_sve_sqabs">;
1512defm SVQNEG : SInstZPZ<"svqneg", "csil", "aarch64_sve_sqneg">;
1513defm SVRECPE : SInstZPZ<"svrecpe", "Ui", "aarch64_sve_urecpe">;
1514defm SVRSQRTE : SInstZPZ<"svrsqrte", "Ui", "aarch64_sve_ursqrte">;
1515}
1516
1517//------------------------------------------------------------------------------
1518
1519multiclass SInstZPZxZ<string name, string types, string pat_v, string pat_n, string intrinsic, list<FlagType> flags=[]> {
1520 def _M : SInst<name # "[_{d}]", pat_v, types, MergeOp1, intrinsic, flags>;
1521 def _X : SInst<name # "[_{d}]", pat_v, types, MergeAny, intrinsic, flags>;
1522 def _Z : SInst<name # "[_{d}]", pat_v, types, MergeZero, intrinsic, flags>;
1523
1524 def _N_M : SInst<name # "[_n_{d}]", pat_n, types, MergeOp1, intrinsic, flags>;
1525 def _N_X : SInst<name # "[_n_{d}]", pat_n, types, MergeAny, intrinsic, flags>;
1526 def _N_Z : SInst<name # "[_n_{d}]", pat_n, types, MergeZero, intrinsic, flags>;
1527}
1528
1529let ArchGuard = "defined(__ARM_FEATURE_SVE2)" in {
1530defm SVQRSHL_S : SInstZPZxZ<"svqrshl", "csil", "dPdx", "dPdK", "aarch64_sve_sqrshl">;
1531defm SVQRSHL_U : SInstZPZxZ<"svqrshl", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_uqrshl">;
1532defm SVQSHL_S : SInstZPZxZ<"svqshl", "csil", "dPdx", "dPdK", "aarch64_sve_sqshl">;
1533defm SVQSHL_U : SInstZPZxZ<"svqshl", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_uqshl">;
1534defm SVRSHL_S : SInstZPZxZ<"svrshl", "csil", "dPdx", "dPdK", "aarch64_sve_srshl">;
1535defm SVRSHL_U : SInstZPZxZ<"svrshl", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_urshl">;
1536defm SVSQADD : SInstZPZxZ<"svsqadd", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_usqadd">;
1537defm SVUQADD : SInstZPZxZ<"svuqadd", "csil", "dPdu", "dPdL", "aarch64_sve_suqadd">;
1538
1539def SVABA_S : SInst<"svaba[_{d}]", "dddd", "csil" , MergeNone, "aarch64_sve_saba">;
1540def SVABA_U : SInst<"svaba[_{d}]", "dddd", "UcUsUiUl", MergeNone, "aarch64_sve_uaba">;
1541def SVQDMULH : SInst<"svqdmulh[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqdmulh">;
1542def SVQRDMULH : SInst<"svqrdmulh[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqrdmulh">;
1543def SVQRDMLAH : SInst<"svqrdmlah[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sqrdmlah">;
1544def SVQRDMLSH : SInst<"svqrdmlsh[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sqrdmlsh">;
1545
1546def SVABA_S_N : SInst<"svaba[_n_{d}]", "ddda", "csil", MergeNone, "aarch64_sve_saba">;
1547def SVABA_U_N : SInst<"svaba[_n_{d}]", "ddda", "UcUsUiUl", MergeNone, "aarch64_sve_uaba">;
1548def SVQDMULH_N : SInst<"svqdmulh[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqdmulh">;
1549def SVQRDMULH_N : SInst<"svqrdmulh[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqrdmulh">;
1550def SVQRDMLAH_N : SInst<"svqrdmlah[_n_{d}]", "ddda", "csil", MergeNone, "aarch64_sve_sqrdmlah">;
1551def SVQRDMLSH_N : SInst<"svqrdmlsh[_n_{d}]", "ddda", "csil", MergeNone, "aarch64_sve_sqrdmlsh">;
1552
1553def SVQDMULH_LANE : SInst<"svqdmulh_lane[_{d}]", "dddi", "sil", MergeNone, "aarch64_sve_sqdmulh_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>;
1554def SVQRDMULH_LANE : SInst<"svqrdmulh_lane[_{d}]", "dddi", "sil", MergeNone, "aarch64_sve_sqrdmulh_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>;
1555def SVQRDMLAH_LANE : SInst<"svqrdmlah_lane[_{d}]", "ddddi", "sil", MergeNone, "aarch64_sve_sqrdmlah_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>;
1556def SVQRDMLSH_LANE : SInst<"svqrdmlsh_lane[_{d}]", "ddddi", "sil", MergeNone, "aarch64_sve_sqrdmlsh_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>;
1557
1558def SVQSHLU_M : SInst<"svqshlu[_n_{d}]", "uPdi", "csil", MergeOp1, "aarch64_sve_sqshlu", [], [ImmCheck<2, ImmCheckShiftLeft, 1>]>;
1559def SVQSHLU_X : SInst<"svqshlu[_n_{d}]", "uPdi", "csil", MergeAny, "aarch64_sve_sqshlu", [], [ImmCheck<2, ImmCheckShiftLeft, 1>]>;
1560def SVQSHLU_Z : SInst<"svqshlu[_n_{d}]", "uPdi", "csil", MergeZero, "aarch64_sve_sqshlu", [], [ImmCheck<2, ImmCheckShiftLeft, 1>]>;
1561def SVRSHR_M_S : SInst<"svrshr[_n_{d}]", "dPdi", "csil", MergeOp1, "aarch64_sve_srshr", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>;
1562def SVRSHR_M_U : SInst<"svrshr[_n_{d}]", "dPdi", "UcUsUiUl", MergeOp1, "aarch64_sve_urshr", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>;
1563def SVRSHR_X_S : SInst<"svrshr[_n_{d}]", "dPdi", "csil", MergeAny, "aarch64_sve_srshr", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>;
1564def SVRSHR_X_U : SInst<"svrshr[_n_{d}]", "dPdi", "UcUsUiUl", MergeAny, "aarch64_sve_urshr", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>;
1565def SVRSHR_Z_S : SInst<"svrshr[_n_{d}]", "dPdi", "csil", MergeZero, "aarch64_sve_srshr", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>;
1566def SVRSHR_Z_U : SInst<"svrshr[_n_{d}]", "dPdi", "UcUsUiUl", MergeZero, "aarch64_sve_urshr", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>;
1567def SVRSRA_S : SInst<"svrsra[_n_{d}]", "dddi", "csil", MergeNone, "aarch64_sve_srsra", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>;
1568def SVRSRA_U : SInst<"svrsra[_n_{d}]", "dddi", "UcUsUiUl", MergeNone, "aarch64_sve_ursra", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>;
1569def SVSLI : SInst<"svsli[_n_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_sli", [], [ImmCheck<2, ImmCheckShiftLeft, 1>]>;
1570def SVSRA_S : SInst<"svsra[_n_{d}]", "dddi", "csil", MergeNone, "aarch64_sve_ssra", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>;
1571def SVSRA_U : SInst<"svsra[_n_{d}]", "dddi", "UcUsUiUl", MergeNone, "aarch64_sve_usra", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>;
1572def SVSRI : SInst<"svsri[_n_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_sri", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>;
1573}
1574
1575////////////////////////////////////////////////////////////////////////////////
1576// SVE2 - Non-widening pairwise arithmetic
1577
1578multiclass SInstPairwise<string name, string types, string intrinsic, list<FlagType> flags=[]> {
1579 def _M : SInst<name # "[_{d}]", "dPdd", types, MergeOp1, intrinsic, flags>;
1580 def _X : SInst<name # "[_{d}]", "dPdd", types, MergeAny, intrinsic, flags>;
1581}
1582
1583let ArchGuard = "defined(__ARM_FEATURE_SVE2)" in {
1584defm SVADDP : SInstPairwise<"svaddp", "csliUcUsUiUl", "aarch64_sve_addp">;
1585defm SVADDP_F : SInstPairwise<"svaddp", "hfd", "aarch64_sve_faddp">;
1586defm SVMAXNMP : SInstPairwise<"svmaxnmp", "hfd", "aarch64_sve_fmaxnmp">;
1587defm SVMAXP_F : SInstPairwise<"svmaxp", "hfd", "aarch64_sve_fmaxp">;
1588defm SVMAXP_S : SInstPairwise<"svmaxp", "csli", "aarch64_sve_smaxp">;
1589defm SVMAXP_U : SInstPairwise<"svmaxp", "UcUsUiUl", "aarch64_sve_umaxp">;
1590defm SVMINNMP : SInstPairwise<"svminnmp", "hfd", "aarch64_sve_fminnmp">;
1591defm SVMINP_F : SInstPairwise<"svminp", "hfd", "aarch64_sve_fminp">;
1592defm SVMINP_S : SInstPairwise<"svminp", "csli", "aarch64_sve_sminp">;
1593defm SVMINP_U : SInstPairwise<"svminp", "UcUsUiUl", "aarch64_sve_uminp">;
1594}
1595
1596////////////////////////////////////////////////////////////////////////////////
1597// SVE2 - Widening pairwise arithmetic
1598
1599let ArchGuard = "defined(__ARM_FEATURE_SVE2)" in {
1600def SVADALP_S_M : SInst<"svadalp[_{d}]", "dPdh", "sil", MergeOp1, "aarch64_sve_sadalp">;
1601def SVADALP_S_X : SInst<"svadalp[_{d}]", "dPdh", "sil", MergeAny, "aarch64_sve_sadalp">;
1602def SVADALP_S_Z : SInst<"svadalp[_{d}]", "dPdh", "sil", MergeZero, "aarch64_sve_sadalp">;
1603
1604def SVADALP_U_M : SInst<"svadalp[_{d}]", "dPdh", "UsUiUl", MergeOp1, "aarch64_sve_uadalp">;
1605def SVADALP_U_X : SInst<"svadalp[_{d}]", "dPdh", "UsUiUl", MergeAny, "aarch64_sve_uadalp">;
1606def SVADALP_U_Z : SInst<"svadalp[_{d}]", "dPdh", "UsUiUl", MergeZero, "aarch64_sve_uadalp">;
1607}
1608
1609////////////////////////////////////////////////////////////////////////////////
1610// SVE2 - Bitwise ternary logical instructions
1611//
1612
1613let ArchGuard = "defined(__ARM_FEATURE_SVE2)" in {
1614def SVBCAX : SInst<"svbcax[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bcax">;
1615def SVBSL : SInst<"svbsl[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl">;
1616def SVBSL1N : SInst<"svbsl1n[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl1n">;
1617def SVBSL2N : SInst<"svbsl2n[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl2n">;
1618def SVEOR3 : SInst<"sveor3[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eor3">;
1619def SVNBSL : SInst<"svnbsl[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_nbsl">;
1620
1621def SVBCAX_N : SInst<"svbcax[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bcax">;
1622def SVBSL_N : SInst<"svbsl[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl">;
1623def SVBSL1N_N : SInst<"svbsl1n[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl1n">;
1624def SVBSL2N_N : SInst<"svbsl2n[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl2n">;
1625def SVEOR3_N : SInst<"sveor3[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_eor3">;
1626def SVNBSL_N : SInst<"svnbsl[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_nbsl">;
1627def SVXAR_N : SInst<"svxar[_n_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_xar", [], [ImmCheck<2, ImmCheckShiftRight, 1>]>;
1628}
1629
1630////////////////////////////////////////////////////////////////////////////////
1631// SVE2 - Large integer arithmetic
1632
1633let ArchGuard = "defined(__ARM_FEATURE_SVE2)" in {
1634def SVADCLB : SInst<"svadclb[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_adclb">;
1635def SVADCLT : SInst<"svadclt[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_adclt">;
1636def SVSBCLB : SInst<"svsbclb[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_sbclb">;
1637def SVSBCLT : SInst<"svsbclt[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_sbclt">;
1638
1639def SVADCLB_N : SInst<"svadclb[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_adclb">;
1640def SVADCLT_N : SInst<"svadclt[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_adclt">;
1641def SVSBCLB_N : SInst<"svsbclb[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_sbclb">;
1642def SVSBCLT_N : SInst<"svsbclt[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_sbclt">;
1643}
1644
1645////////////////////////////////////////////////////////////////////////////////
1646// SVE2 - Multiplication by indexed elements
1647
1648let ArchGuard = "defined(__ARM_FEATURE_SVE2)" in {
1649def SVMLA_LANE_2 : SInst<"svmla_lane[_{d}]", "ddddi", "silUsUiUl", MergeNone, "aarch64_sve_mla_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>;
1650def SVMLS_LANE_2 : SInst<"svmls_lane[_{d}]", "ddddi", "silUsUiUl", MergeNone, "aarch64_sve_mls_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>;
1651def SVMUL_LANE_2 : SInst<"svmul_lane[_{d}]", "dddi", "silUsUiUl", MergeNone, "aarch64_sve_mul_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>;
1652}
1653
1654////////////////////////////////////////////////////////////////////////////////
1655// SVE2 - Uniform complex integer arithmetic
1656let ArchGuard = "defined(__ARM_FEATURE_SVE2)" in {
1657def SVCADD : SInst<"svcadd[_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_cadd_x", [], [ImmCheck<2, ImmCheckComplexRot90_270>]>;
1658def SVSQCADD : SInst<"svqcadd[_{d}]", "dddi", "csil", MergeNone, "aarch64_sve_sqcadd_x", [], [ImmCheck<2, ImmCheckComplexRot90_270>]>;
1659def SVCMLA : SInst<"svcmla[_{d}]", "ddddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmla_x", [], [ImmCheck<3, ImmCheckComplexRotAll90>]>;
1660def SVCMLA_LANE_X : SInst<"svcmla_lane[_{d}]", "ddddii", "siUsUi", MergeNone, "aarch64_sve_cmla_lane_x", [], [ImmCheck<3, ImmCheckLaneIndexCompRotate, 2>,
1661 ImmCheck<4, ImmCheckComplexRotAll90>]>;
1662def SVSQRDCMLAH_X : SInst<"svqrdcmlah[_{d}]", "ddddi", "csil", MergeNone, "aarch64_sve_sqrdcmlah_x", [], [ImmCheck<3, ImmCheckComplexRotAll90>]>;
1663def SVSQRDCMLAH_LANE_X : SInst<"svqrdcmlah_lane[_{d}]", "ddddii", "si", MergeNone, "aarch64_sve_sqrdcmlah_lane_x", [], [ImmCheck<3, ImmCheckLaneIndexCompRotate, 2>,
1664 ImmCheck<4, ImmCheckComplexRotAll90>]>;
1665}
1666
1667////////////////////////////////////////////////////////////////////////////////
1668// SVE2 - Widening DSP operations
1669
1670multiclass SInstWideDSPAcc<string name, string types, string intrinsic> {
1671 def : SInst<name # "[_{d}]", "ddhh", types, MergeNone, intrinsic>;
1672 def _N : SInst<name # "[_n_{d}]", "ddhR", types, MergeNone, intrinsic>;
1673}
1674
1675multiclass SInstWideDSPLong<string name, string types, string intrinsic> {
1676 def : SInst<name # "[_{d}]", "dhh", types, MergeNone, intrinsic>;
1677 def _N : SInst<name # "[_n_{d}]", "dhR", types, MergeNone, intrinsic>;
1678}
1679
1680multiclass SInstWideDSPWide<string name, string types, string intrinsic> {
1681 def : SInst<name # "[_{d}]", "ddh", types, MergeNone, intrinsic>;
1682 def _N : SInst<name # "[_n_{d}]", "ddR", types, MergeNone, intrinsic>;
1683}
1684
1685let ArchGuard = "defined(__ARM_FEATURE_SVE2)" in {
1686defm SVABALB_S : SInstWideDSPAcc<"svabalb", "sil", "aarch64_sve_sabalb">;
1687defm SVABALB_U : SInstWideDSPAcc<"svabalb", "UsUiUl", "aarch64_sve_uabalb">;
1688defm SVABALT_S : SInstWideDSPAcc<"svabalt", "sil", "aarch64_sve_sabalt">;
1689defm SVABALT_U : SInstWideDSPAcc<"svabalt", "UsUiUl", "aarch64_sve_uabalt">;
1690defm SVMLALB_S : SInstWideDSPAcc<"svmlalb", "sil", "aarch64_sve_smlalb">;
1691defm SVMLALB_U : SInstWideDSPAcc<"svmlalb", "UsUiUl", "aarch64_sve_umlalb">;
1692defm SVMLALT_S : SInstWideDSPAcc<"svmlalt", "sil", "aarch64_sve_smlalt">;
1693defm SVMLALT_U : SInstWideDSPAcc<"svmlalt", "UsUiUl", "aarch64_sve_umlalt">;
1694defm SVMLSLB_S : SInstWideDSPAcc<"svmlslb", "sil", "aarch64_sve_smlslb">;
1695defm SVMLSLB_U : SInstWideDSPAcc<"svmlslb", "UsUiUl", "aarch64_sve_umlslb">;
1696defm SVMLSLT_S : SInstWideDSPAcc<"svmlslt", "sil", "aarch64_sve_smlslt">;
1697defm SVMLSLT_U : SInstWideDSPAcc<"svmlslt", "UsUiUl", "aarch64_sve_umlslt">;
1698defm SVQDMLALB : SInstWideDSPAcc<"svqdmlalb", "sil", "aarch64_sve_sqdmlalb">;
1699defm SVQDMLALT : SInstWideDSPAcc<"svqdmlalt", "sil", "aarch64_sve_sqdmlalt">;
1700defm SVQDMLSLB : SInstWideDSPAcc<"svqdmlslb", "sil", "aarch64_sve_sqdmlslb">;
1701defm SVQDMLSLT : SInstWideDSPAcc<"svqdmlslt", "sil", "aarch64_sve_sqdmlslt">;
1702
1703defm SVABDLB_S : SInstWideDSPLong<"svabdlb", "sil", "aarch64_sve_sabdlb">;
1704defm SVABDLB_U : SInstWideDSPLong<"svabdlb", "UsUiUl", "aarch64_sve_uabdlb">;
1705defm SVABDLT_S : SInstWideDSPLong<"svabdlt", "sil", "aarch64_sve_sabdlt">;
1706defm SVABDLT_U : SInstWideDSPLong<"svabdlt", "UsUiUl", "aarch64_sve_uabdlt">;
1707defm SVADDLB_S : SInstWideDSPLong<"svaddlb", "sil", "aarch64_sve_saddlb">;
1708defm SVADDLB_U : SInstWideDSPLong<"svaddlb", "UsUiUl", "aarch64_sve_uaddlb">;
1709defm SVADDLT_S : SInstWideDSPLong<"svaddlt", "sil", "aarch64_sve_saddlt">;
1710defm SVADDLT_U : SInstWideDSPLong<"svaddlt", "UsUiUl", "aarch64_sve_uaddlt">;
1711defm SVMULLB_S : SInstWideDSPLong<"svmullb", "sil", "aarch64_sve_smullb">;
1712defm SVMULLB_U : SInstWideDSPLong<"svmullb", "UsUiUl", "aarch64_sve_umullb">;
1713defm SVMULLT_S : SInstWideDSPLong<"svmullt", "sil", "aarch64_sve_smullt">;
1714defm SVMULLT_U : SInstWideDSPLong<"svmullt", "UsUiUl", "aarch64_sve_umullt">;
1715defm SVQDMULLB : SInstWideDSPLong<"svqdmullb", "sil", "aarch64_sve_sqdmullb">;
1716defm SVQDMULLT : SInstWideDSPLong<"svqdmullt", "sil", "aarch64_sve_sqdmullt">;
1717defm SVSUBLB_S : SInstWideDSPLong<"svsublb", "sil", "aarch64_sve_ssublb">;
1718defm SVSUBLB_U : SInstWideDSPLong<"svsublb", "UsUiUl", "aarch64_sve_usublb">;
1719defm SVSUBLT_S : SInstWideDSPLong<"svsublt", "sil", "aarch64_sve_ssublt">;
1720defm SVSUBLT_U : SInstWideDSPLong<"svsublt", "UsUiUl", "aarch64_sve_usublt">;
1721
1722defm SVADDWB_S : SInstWideDSPWide<"svaddwb", "sil", "aarch64_sve_saddwb">;
1723defm SVADDWB_U : SInstWideDSPWide<"svaddwb", "UsUiUl", "aarch64_sve_uaddwb">;
1724defm SVADDWT_S : SInstWideDSPWide<"svaddwt", "sil", "aarch64_sve_saddwt">;
1725defm SVADDWT_U : SInstWideDSPWide<"svaddwt", "UsUiUl", "aarch64_sve_uaddwt">;
1726defm SVSUBWB_S : SInstWideDSPWide<"svsubwb", "sil", "aarch64_sve_ssubwb">;
1727defm SVSUBWB_U : SInstWideDSPWide<"svsubwb", "UsUiUl", "aarch64_sve_usubwb">;
1728defm SVSUBWT_S : SInstWideDSPWide<"svsubwt", "sil", "aarch64_sve_ssubwt">;
1729defm SVSUBWT_U : SInstWideDSPWide<"svsubwt", "UsUiUl", "aarch64_sve_usubwt">;
1730
1731def SVSHLLB_S_N : SInst<"svshllb[_n_{d}]", "dhi", "sil", MergeNone, "aarch64_sve_sshllb", [], [ImmCheck<1, ImmCheckShiftLeft, 0>]>;
1732def SVSHLLB_U_N : SInst<"svshllb[_n_{d}]", "dhi", "UsUiUl", MergeNone, "aarch64_sve_ushllb", [], [ImmCheck<1, ImmCheckShiftLeft, 0>]>;
1733def SVSHLLT_S_N : SInst<"svshllt[_n_{d}]", "dhi", "sil", MergeNone, "aarch64_sve_sshllt", [], [ImmCheck<1, ImmCheckShiftLeft, 0>]>;
1734def SVSHLLT_U_N : SInst<"svshllt[_n_{d}]", "dhi", "UsUiUl", MergeNone, "aarch64_sve_ushllt", [], [ImmCheck<1, ImmCheckShiftLeft, 0>]>;
1735
1736def SVMOVLB_S_N : SInst<"svmovlb[_{d}]", "dh", "sil", MergeNone>;
1737def SVMOVLB_U_N : SInst<"svmovlb[_{d}]", "dh", "UsUiUl", MergeNone>;
1738def SVMOVLT_S_N : SInst<"svmovlt[_{d}]", "dh", "sil", MergeNone>;
1739def SVMOVLT_U_N : SInst<"svmovlt[_{d}]", "dh", "UsUiUl", MergeNone>;
1740
1741def SVMLALB_S_LANE : SInst<"svmlalb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlalb_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>;
1742def SVMLALB_U_LANE : SInst<"svmlalb_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlalb_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>;
1743def SVMLALT_S_LANE : SInst<"svmlalt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlalt_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>;
1744def SVMLALT_U_LANE : SInst<"svmlalt_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlalt_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>;
1745def SVMLSLB_S_LANE : SInst<"svmlslb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlslb_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>;
1746def SVMLSLB_U_LANE : SInst<"svmlslb_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlslb_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>;
1747def SVMLSLT_S_LANE : SInst<"svmlslt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlslt_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>;
1748def SVMLSLT_U_LANE : SInst<"svmlslt_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlslt_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>;
1749def SVMULLB_S_LANE : SInst<"svmullb_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_smullb_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>;
1750def SVMULLB_U_LANE : SInst<"svmullb_lane[_{d}]", "dhhi", "UiUl", MergeNone, "aarch64_sve_umullb_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>;
1751def SVMULLT_S_LANE : SInst<"svmullt_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_smullt_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>;
1752def SVMULLT_U_LANE : SInst<"svmullt_lane[_{d}]", "dhhi", "UiUl", MergeNone, "aarch64_sve_umullt_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>;
1753def SVQDMLALB_LANE : SInst<"svqdmlalb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlalb_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>;
1754def SVQDMLALT_LANE : SInst<"svqdmlalt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlalt_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>;
1755def SVQDMLSLB_LANE : SInst<"svqdmlslb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlslb_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>;
1756def SVQDMLSLT_LANE : SInst<"svqdmlslt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlslt_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>;
1757def SVQDMULLB_LANE : SInst<"svqdmullb_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_sqdmullb_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>;
1758def SVQDMULLT_LANE : SInst<"svqdmullt_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_sqdmullt_lane", [], [ImmCheck<2, ImmCheckLaneIndex, 1>]>;
1759}
1760
1761////////////////////////////////////////////////////////////////////////////////
1762// SVE2 - Narrowing DSP operations
1763
1764let ArchGuard = "defined(__ARM_FEATURE_SVE2)" in {
1765def SVADDHNB : SInst<"svaddhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_addhnb">;
1766def SVADDHNT : SInst<"svaddhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_addhnt">;
1767def SVRADDHNB : SInst<"svraddhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_raddhnb">;
1768def SVRADDHNT : SInst<"svraddhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_raddhnt">;
1769def SVRSUBHNB : SInst<"svrsubhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnb">;
1770def SVRSUBHNT : SInst<"svrsubhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnt">;
1771def SVSUBHNB : SInst<"svsubhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_subhnb">;
1772def SVSUBHNT : SInst<"svsubhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_subhnt">;
1773
1774def SVADDHNB_N : SInst<"svaddhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_addhnb">;
1775def SVADDHNT_N : SInst<"svaddhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_addhnt">;
1776def SVRADDHNB_N : SInst<"svraddhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_raddhnb">;
1777def SVRADDHNT_N : SInst<"svraddhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_raddhnt">;
1778def SVRSUBHNB_N : SInst<"svrsubhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnb">;
1779def SVRSUBHNT_N : SInst<"svrsubhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnt">;
1780def SVSUBHNB_N : SInst<"svsubhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_subhnb">;
1781def SVSUBHNT_N : SInst<"svsubhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_subhnt">;
1782
1783def SVSHRNB : SInst<"svshrnb[_n_{d}]", "hdi", "silUsUiUl", MergeNone, "aarch64_sve_shrnb", [], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>;
1784def SVRSHRNB : SInst<"svrshrnb[_n_{d}]", "hdi", "silUsUiUl", MergeNone, "aarch64_sve_rshrnb", [], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>;
1785def SVQSHRUNB : SInst<"svqshrunb[_n_{d}]", "edi", "sil", MergeNone, "aarch64_sve_sqshrunb", [], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>;
1786def SVQRSHRUNB : SInst<"svqrshrunb[_n_{d}]", "edi", "sil", MergeNone, "aarch64_sve_sqrshrunb", [], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>;
1787def SVQSHRNB_S : SInst<"svqshrnb[_n_{d}]", "hdi", "sil", MergeNone, "aarch64_sve_sqshrnb", [], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>;
1788def SVQSHRNB_U : SInst<"svqshrnb[_n_{d}]", "hdi", "UsUiUl", MergeNone, "aarch64_sve_uqshrnb", [], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>;
1789def SVQRSHRNB_S : SInst<"svqrshrnb[_n_{d}]", "hdi", "sil", MergeNone, "aarch64_sve_sqrshrnb", [], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>;
1790def SVQRSHRNB_U : SInst<"svqrshrnb[_n_{d}]", "hdi", "UsUiUl", MergeNone, "aarch64_sve_uqrshrnb", [], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>;
1791
1792def SVSHRNT : SInst<"svshrnt[_n_{d}]", "hhdi", "silUsUiUl", MergeNone, "aarch64_sve_shrnt", [], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>;
1793def SVRSHRNT : SInst<"svrshrnt[_n_{d}]", "hhdi", "silUsUiUl", MergeNone, "aarch64_sve_rshrnt", [], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>;
1794def SVQSHRUNT : SInst<"svqshrunt[_n_{d}]", "eedi", "sil", MergeNone, "aarch64_sve_sqshrunt", [], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>;
1795def SVQRSHRUNT : SInst<"svqrshrunt[_n_{d}]", "eedi", "sil", MergeNone, "aarch64_sve_sqrshrunt", [], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>;
1796def SVQSHRNT_S : SInst<"svqshrnt[_n_{d}]", "hhdi", "sil", MergeNone, "aarch64_sve_sqshrnt", [], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>;
1797def SVQSHRNT_U : SInst<"svqshrnt[_n_{d}]", "hhdi", "UsUiUl", MergeNone, "aarch64_sve_uqshrnt", [], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>;
1798def SVQRSHRNT_S : SInst<"svqrshrnt[_n_{d}]", "hhdi", "sil", MergeNone, "aarch64_sve_sqrshrnt", [], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>;
1799def SVQRSHRNT_U : SInst<"svqrshrnt[_n_{d}]", "hhdi", "UsUiUl", MergeNone, "aarch64_sve_uqrshrnt", [], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>;
1800}
1801////////////////////////////////////////////////////////////////////////////////
1802// SVE2 - Unary narrowing operations
1803
1804let ArchGuard = "defined(__ARM_FEATURE_SVE2)" in {
1805def SVQXTNB_S : SInst<"svqxtnb[_{d}]", "hd", "sil", MergeNone, "aarch64_sve_sqxtnb">;
1806def SVQXTNB_U : SInst<"svqxtnb[_{d}]", "hd", "UsUiUl", MergeNone, "aarch64_sve_uqxtnb">;
1807def SVQXTUNB_S : SInst<"svqxtunb[_{d}]", "ed", "sil", MergeNone, "aarch64_sve_sqxtunb">;
1808
1809def SVQXTNT_S : SInst<"svqxtnt[_{d}]", "hhd", "sil", MergeNone, "aarch64_sve_sqxtnt">;
1810def SVQXTNT_U : SInst<"svqxtnt[_{d}]", "hhd", "UsUiUl", MergeNone, "aarch64_sve_uqxtnt">;
1811def SVQXTUNT_S : SInst<"svqxtunt[_{d}]", "eed", "sil", MergeNone, "aarch64_sve_sqxtunt">;
1812}
1813
1814////////////////////////////////////////////////////////////////////////////////
1815// SVE2 - Widening complex integer arithmetic
1816
1817let ArchGuard = "defined(__ARM_FEATURE_SVE2)" in {
1818defm SVADDLBT : SInstWideDSPLong<"svaddlbt", "sil", "aarch64_sve_saddlbt">;
1819defm SVSUBLBT : SInstWideDSPLong<"svsublbt", "sil", "aarch64_sve_ssublbt">;
1820defm SVSUBLTB : SInstWideDSPLong<"svsubltb", "sil", "aarch64_sve_ssubltb">;
1821
1822defm SVQDMLALBT : SInstWideDSPAcc<"svqdmlalbt", "sil", "aarch64_sve_sqdmlalbt">;
1823defm SVQDMLSLBT : SInstWideDSPAcc<"svqdmlslbt", "sil", "aarch64_sve_sqdmlslbt">;
1824}
1825
1826////////////////////////////////////////////////////////////////////////////////
1827// SVE2 - Non-temporal gather/scatter
1828let ArchGuard = "defined(__ARM_FEATURE_SVE2)" in {
1829// Non-temporal gather load one vector (vector base)
1830def SVLDNT1_GATHER_BASES_U : MInst<"svldnt1_gather[_{2}base]_{0}", "dPu", "ilUiUlfd", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldnt1_gather_scalar_offset">;
1831def SVLDNT1SB_GATHER_BASES_U : MInst<"svldnt1sb_gather[_{2}base]_{0}", "dPu", "ilUiUl", [IsGatherLoad], MemEltTyInt8, "aarch64_sve_ldnt1_gather_scalar_offset">;
1832def SVLDNT1UB_GATHER_BASES_U : MInst<"svldnt1ub_gather[_{2}base]_{0}", "dPu", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldnt1_gather_scalar_offset">;
1833def SVLDNT1SH_GATHER_BASES_U : MInst<"svldnt1sh_gather[_{2}base]_{0}", "dPu", "ilUiUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldnt1_gather_scalar_offset">;
1834def SVLDNT1UH_GATHER_BASES_U : MInst<"svldnt1uh_gather[_{2}base]_{0}", "dPu", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnt1_gather_scalar_offset">;
1835def SVLDNT1SW_GATHER_BASES_U : MInst<"svldnt1sw_gather[_{2}base]_{0}", "dPu", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ldnt1_gather_scalar_offset">;
1836def SVLDNT1UW_GATHER_BASES_U : MInst<"svldnt1uw_gather[_{2}base]_{0}", "dPu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnt1_gather_scalar_offset">;
1837
1838// Non-temporal gather load one vector (scalar base, signed vector offset in bytes)
1839def SVLDNT1_GATHER_64B_OFFSETS_S : MInst<"svldnt1_gather_[{3}]offset[_{0}]", "dPcx", "lUld", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldnt1_gather">;
1840def SVLDNT1SB_GATHER_64B_OFFSETS_S : MInst<"svldnt1sb_gather_[{3}]offset_{0}", "dPSx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldnt1_gather">;
1841def SVLDNT1UB_GATHER_64B_OFFSETS_S : MInst<"svldnt1ub_gather_[{3}]offset_{0}", "dPWx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldnt1_gather">;
1842def SVLDNT1SH_GATHER_64B_OFFSETS_S : MInst<"svldnt1sh_gather_[{3}]offset_{0}", "dPTx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldnt1_gather">;
1843def SVLDNT1UH_GATHER_64B_OFFSETS_S : MInst<"svldnt1uh_gather_[{3}]offset_{0}", "dPXx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnt1_gather">;
1844def SVLDNT1SW_GATHER_64B_OFFSETS_S : MInst<"svldnt1sw_gather_[{3}]offset_{0}", "dPUx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ldnt1_gather">;
1845def SVLDNT1UW_GATHER_64B_OFFSETS_S : MInst<"svldnt1uw_gather_[{3}]offset_{0}", "dPYx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnt1_gather">;
1846
1847// Non-temporal gather load one vector (scalar base, unsigned vector offset in bytes)
1848def SVLDNT1_GATHER_64B_OFFSETS_U : MInst<"svldnt1_gather_[{3}]offset[_{0}]", "dPcu", "lUld", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldnt1_gather">;
1849def SVLDNT1SB_GATHER_64B_OFFSETS_U : MInst<"svldnt1sb_gather_[{3}]offset_{0}", "dPSu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldnt1_gather">;
1850def SVLDNT1UB_GATHER_64B_OFFSETS_U : MInst<"svldnt1ub_gather_[{3}]offset_{0}", "dPWu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldnt1_gather">;
1851def SVLDNT1SH_GATHER_64B_OFFSETS_U : MInst<"svldnt1sh_gather_[{3}]offset_{0}", "dPTu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldnt1_gather">;
1852def SVLDNT1UH_GATHER_64B_OFFSETS_U : MInst<"svldnt1uh_gather_[{3}]offset_{0}", "dPXu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnt1_gather">;
1853def SVLDNT1SW_GATHER_64B_OFFSETS_U : MInst<"svldnt1sw_gather_[{3}]offset_{0}", "dPUu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ldnt1_gather">;
1854def SVLDNT1UW_GATHER_64B_OFFSETS_U : MInst<"svldnt1uw_gather_[{3}]offset_{0}", "dPYu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnt1_gather">;
1855
1856def SVLDNT1_GATHER_32B_OFFSETS_U : MInst<"svldnt1_gather_[{3}]offset[_{0}]", "dPcu", "iUif", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldnt1_gather_uxtw">;
1857def SVLDNT1SB_GATHER_32B_OFFSETS_U : MInst<"svldnt1sb_gather_[{3}]offset_{0}", "dPSu", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldnt1_gather_uxtw">;
1858def SVLDNT1UB_GATHER_32B_OFFSETS_U : MInst<"svldnt1ub_gather_[{3}]offset_{0}", "dPWu", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldnt1_gather_uxtw">;
1859def SVLDNT1SH_GATHER_32B_OFFSETS_U : MInst<"svldnt1sh_gather_[{3}]offset_{0}", "dPTu", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldnt1_gather_uxtw">;
1860def SVLDNT1UH_GATHER_32B_OFFSETS_U : MInst<"svldnt1uh_gather_[{3}]offset_{0}", "dPXu", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnt1_gather_uxtw">;
1861
1862// Non-temporal gather load one vector (vector base, scalar offset in bytes)
1863def SVLDNT1_GATHER_OFFSET_S : MInst<"svldnt1_gather[_{2}base]_offset_{0}", "dPul", "ilUiUlfd", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldnt1_gather_scalar_offset">;
1864def SVLDNT1SB_GATHER_OFFSET_S : MInst<"svldnt1sb_gather[_{2}base]_offset_{0}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldnt1_gather_scalar_offset">;
1865def SVLDNT1UB_GATHER_OFFSET_S : MInst<"svldnt1ub_gather[_{2}base]_offset_{0}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldnt1_gather_scalar_offset">;
1866def SVLDNT1SH_GATHER_OFFSET_S : MInst<"svldnt1sh_gather[_{2}base]_offset_{0}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldnt1_gather_scalar_offset">;
1867def SVLDNT1UH_GATHER_OFFSET_S : MInst<"svldnt1uh_gather[_{2}base]_offset_{0}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnt1_gather_scalar_offset">;
1868def SVLDNT1SW_GATHER_OFFSET_S : MInst<"svldnt1sw_gather[_{2}base]_offset_{0}", "dPul", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ldnt1_gather_scalar_offset">;
1869def SVLDNT1UW_GATHER_OFFSET_S : MInst<"svldnt1uw_gather[_{2}base]_offset_{0}", "dPul", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnt1_gather_scalar_offset">;
1870
1871// Non-temporal gather load one vector (scalar base, signed vector index)
1872def SVLDNT1_GATHER_64B_INDICES_S : MInst<"svldnt1_gather_[{3}]index[_{0}]", "dPcx", "lUld", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldnt1_gather_index">;
1873def SVLDNT1SH_GATHER_64B_INDICES_S : MInst<"svldnt1sh_gather_[{3}]index_{0}", "dPTx", "lUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldnt1_gather_index">;
1874def SVLDNT1UH_GATHER_64B_INDICES_S : MInst<"svldnt1uh_gather_[{3}]index_{0}", "dPXx", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnt1_gather_index">;
1875def SVLDNT1SW_GATHER_64B_INDICES_S : MInst<"svldnt1sw_gather_[{3}]index_{0}", "dPUx", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ldnt1_gather_index">;
1876def SVLDNT1UW_GATHER_64B_INDICES_S : MInst<"svldnt1uw_gather_[{3}]index_{0}", "dPYx", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnt1_gather_index">;
1877
1878// Non temporal gather load one vector (scalar base, unsigned vector index)
1879def SVLDNT1_GATHER_64B_INDICES_U : MInst<"svldnt1_gather_[{3}]index[_{0}]", "dPcu", "lUld", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldnt1_gather_index">;
1880def SVLDNT1SH_GATHER_64B_INDICES_U : MInst<"svldnt1sh_gather_[{3}]index_{0}", "dPTu", "lUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldnt1_gather_index">;
1881def SVLDNT1UH_GATHER_64B_INDICES_U : MInst<"svldnt1uh_gather_[{3}]index_{0}", "dPXu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnt1_gather_index">;
1882def SVLDNT1SW_GATHER_64B_INDICES_U : MInst<"svldnt1sw_gather_[{3}]index_{0}", "dPUu", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ldnt1_gather_index">;
1883def SVLDNT1UW_GATHER_64B_INDICES_U : MInst<"svldnt1uw_gather_[{3}]index_{0}", "dPYu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnt1_gather_index">;
1884
1885// Non-temporal gather load one vector (vector base, signed scalar index)
1886def SVLDNT1_GATHER_INDEX_S : MInst<"svldnt1_gather[_{2}base]_index_{0}", "dPul", "ilUiUlfd", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldnt1_gather_scalar_offset">;
1887def SVLDNT1SH_GATHER_INDEX_S : MInst<"svldnt1sh_gather[_{2}base]_index_{0}", "dPul", "ilUiUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldnt1_gather_scalar_offset">;
1888def SVLDNT1UH_GATHER_INDEX_S : MInst<"svldnt1uh_gather[_{2}base]_index_{0}", "dPul", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnt1_gather_scalar_offset">;
1889def SVLDNT1SW_GATHER_INDEX_S : MInst<"svldnt1sw_gather[_{2}base]_index_{0}", "dPul", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ldnt1_gather_scalar_offset">;
1890def SVLDNT1UW_GATHER_INDEX_S : MInst<"svldnt1uw_gather[_{2}base]_index_{0}", "dPul", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnt1_gather_scalar_offset">;
1891
1892// Non-temporal scatter store one vector (vector base)
1893def SVSTNT1_SCATTER_BASES_U : MInst<"svstnt1_scatter[_{2}base_{d}]", "vPud", "ilUiUlfd", [IsScatterStore], MemEltTyDefault, "aarch64_sve_stnt1_scatter_scalar_offset">;
1894def SVSTNT1B_SCATTER_BASES_U : MInst<"svstnt1b_scatter[_{2}base_{d}]", "vPud", "ilUiUl", [IsScatterStore], MemEltTyInt8, "aarch64_sve_stnt1_scatter_scalar_offset">;
1895def SVSTNT1H_SCATTER_BASES_U : MInst<"svstnt1h_scatter[_{2}base_{d}]", "vPud", "ilUiUl", [IsScatterStore], MemEltTyInt16, "aarch64_sve_stnt1_scatter_scalar_offset">;
1896def SVSTNT1W_SCATTER_BASES_U : MInst<"svstnt1w_scatter[_{2}base_{d}]", "vPud", "lUl", [IsScatterStore], MemEltTyInt32, "aarch64_sve_stnt1_scatter_scalar_offset">;
1897
1898// Non-temporal scatter store one vector (scalar base, signed vector offset in bytes)
1899def SVSTNT1_SCATTER_64B_OFFSETS_S : MInst<"svstnt1_scatter_[{3}]offset[_{d}]", "vPpxd", "lUld", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_stnt1_scatter">;
1900def SVSTNT1B_SCATTER_64B_OFFSETS_SS : MInst<"svstnt1b_scatter_[{3}]offset[_{d}]", "vPAxd", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_stnt1_scatter">;
1901def SVSTNT1B_SCATTER_64B_OFFSETS_SU : MInst<"svstnt1b_scatter_[{3}]offset[_{d}]", "vPExd", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_stnt1_scatter">;
1902def SVSTNT1H_SCATTER_64B_OFFSETS_SS : MInst<"svstnt1h_scatter_[{3}]offset[_{d}]", "vPBxd", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_stnt1_scatter">;
1903def SVSTNT1H_SCATTER_64B_OFFSETS_SU : MInst<"svstnt1h_scatter_[{3}]offset[_{d}]", "vPFxd", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_stnt1_scatter">;
1904def SVSTNT1W_SCATTER_64B_OFFSETS_SS : MInst<"svstnt1w_scatter_[{3}]offset[_{d}]", "vPCxd", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_stnt1_scatter">;
1905def SVSTNT1W_SCATTER_64B_OFFSETS_SU : MInst<"svstnt1w_scatter_[{3}]offset[_{d}]", "vPGxd", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_stnt1_scatter">;
1906
1907// Non-temporal scatter store one vector (scalar base, unsigned vector offset in bytes)
1908def SVSTNT1_SCATTER_64B_OFFSETS_U : MInst<"svstnt1_scatter_[{3}]offset[_{d}]", "vPpud", "lUld", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_stnt1_scatter">;
1909def SVSTNT1B_SCATTER_64B_OFFSETS_US : MInst<"svstnt1b_scatter_[{3}]offset[_{d}]", "vPAud", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_stnt1_scatter">;
1910def SVSTNT1B_SCATTER_64B_OFFSETS_UU : MInst<"svstnt1b_scatter_[{3}]offset[_{d}]", "vPEud", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_stnt1_scatter">;
1911def SVSTNT1H_SCATTER_64B_OFFSETS_US : MInst<"svstnt1h_scatter_[{3}]offset[_{d}]", "vPBud", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_stnt1_scatter">;
1912def SVSTNT1H_SCATTER_64B_OFFSETS_UU : MInst<"svstnt1h_scatter_[{3}]offset[_{d}]", "vPFud", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_stnt1_scatter">;
1913def SVSTNT1W_SCATTER_64B_OFFSETS_US : MInst<"svstnt1w_scatter_[{3}]offset[_{d}]", "vPCud", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_stnt1_scatter">;
1914def SVSTNT1W_SCATTER_64B_OFFSETS_UU : MInst<"svstnt1w_scatter_[{3}]offset[_{d}]", "vPGud", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_stnt1_scatter">;
1915
1916def SVSTNT1_SCATTER_32B_OFFSETS_U : MInst<"svstnt1_scatter_[{3}]offset[_{d}]", "vPpud", "iUif", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_stnt1_scatter_uxtw">;
1917def SVSTNT1B_SCATTER_32B_OFFSETS_US : MInst<"svstnt1b_scatter_[{3}]offset[_{d}]", "vPAud", "i", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_stnt1_scatter_uxtw">;
1918def SVSTNT1B_SCATTER_32B_OFFSETS_UU : MInst<"svstnt1b_scatter_[{3}]offset[_{d}]", "vPEud", "Ui", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_stnt1_scatter_uxtw">;
1919def SVSTNT1H_SCATTER_32B_OFFSETS_US : MInst<"svstnt1h_scatter_[{3}]offset[_{d}]", "vPBud", "i", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_stnt1_scatter_uxtw">;
1920def SVSTNT1H_SCATTER_32B_OFFSETS_UU : MInst<"svstnt1h_scatter_[{3}]offset[_{d}]", "vPFud", "Ui", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_stnt1_scatter_uxtw">;
1921
1922// Non-temporal scatter store one vector (vector base, scalar offset in bytes)
1923def SVSTNT1_SCATTER_OFFSET_S : MInst<"svstnt1_scatter[_{2}base]_offset[_{d}]", "vPuld", "ilUiUlfd", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_stnt1_scatter_scalar_offset">;
1924def SVSTNT1B_SCATTER_OFFSET_S : MInst<"svstnt1b_scatter[_{2}base]_offset[_{d}]", "vPuld", "ilUiUl", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_stnt1_scatter_scalar_offset">;
1925def SVSTNT1H_SCATTER_OFFSET_S : MInst<"svstnt1h_scatter[_{2}base]_offset[_{d}]", "vPuld", "ilUiUl", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_stnt1_scatter_scalar_offset">;
1926def SVSTNT1W_SCATTER_OFFSET_S : MInst<"svstnt1w_scatter[_{2}base]_offset[_{d}]", "vPuld", "lUl", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_stnt1_scatter_scalar_offset">;
1927
1928// Non-temporal scatter store one vector (scalar base, signed vector index)
1929def SVSTNT1_SCATTER_INDICES_S : MInst<"svstnt1_scatter_[{3}]index[_{d}]", "vPpxd", "lUld", [IsScatterStore], MemEltTyDefault, "aarch64_sve_stnt1_scatter_index">;
1930def SVSTNT1H_SCATTER_INDICES_SS : MInst<"svstnt1h_scatter_[{3}]index[_{d}]", "vPBxd", "l", [IsScatterStore], MemEltTyInt16, "aarch64_sve_stnt1_scatter_index">;
1931def SVSTNT1H_SCATTER_INDICES_SU : MInst<"svstnt1h_scatter_[{3}]index[_{d}]", "vPFxd", "Ul", [IsScatterStore], MemEltTyInt16, "aarch64_sve_stnt1_scatter_index">;
1932def SVSTNT1W_SCATTER_INDICES_SS : MInst<"svstnt1w_scatter_[{3}]index[_{d}]", "vPCxd", "l", [IsScatterStore], MemEltTyInt32, "aarch64_sve_stnt1_scatter_index">;
1933def SVSTNT1W_SCATTER_INDICES_SU : MInst<"svstnt1w_scatter_[{3}]index[_{d}]", "vPGxd", "Ul", [IsScatterStore], MemEltTyInt32, "aarch64_sve_stnt1_scatter_index">;
1934
1935// Non-temporal scatter store one vector (scalar base, unsigned vector index)
1936def SVSTNT1_SCATTER_INDICES_U : MInst<"svstnt1_scatter_[{3}]index[_{d}]", "vPpud", "lUld", [IsScatterStore], MemEltTyDefault, "aarch64_sve_stnt1_scatter_index">;
1937def SVSTNT1H_SCATTER_INDICES_US : MInst<"svstnt1h_scatter_[{3}]index[_{d}]", "vPBud", "l", [IsScatterStore], MemEltTyInt16, "aarch64_sve_stnt1_scatter_index">;
1938def SVSTNT1H_SCATTER_INDICES_UU : MInst<"svstnt1h_scatter_[{3}]index[_{d}]", "vPFud", "Ul", [IsScatterStore], MemEltTyInt16, "aarch64_sve_stnt1_scatter_index">;
1939def SVSTNT1W_SCATTER_INDICES_US : MInst<"svstnt1w_scatter_[{3}]index[_{d}]", "vPCud", "l", [IsScatterStore], MemEltTyInt32, "aarch64_sve_stnt1_scatter_index">;
1940def SVSTNT1W_SCATTER_INDICES_UU : MInst<"svstnt1w_scatter_[{3}]index[_{d}]", "vPGud", "Ul", [IsScatterStore], MemEltTyInt32, "aarch64_sve_stnt1_scatter_index">;
1941
1942// Non-temporal scatter store one vector (vector base, signed scalar index)
1943def SVSTNT1_SCATTER_INDEX_S : MInst<"svstnt1_scatter[_{2}base]_index[_{d}]", "vPuld", "ilUiUlfd", [IsScatterStore], MemEltTyDefault, "aarch64_sve_stnt1_scatter_scalar_offset">;
1944def SVSTNT1H_SCATTER_INDEX_S : MInst<"svstnt1h_scatter[_{2}base]_index[_{d}]", "vPuld", "ilUiUl", [IsScatterStore], MemEltTyInt16, "aarch64_sve_stnt1_scatter_scalar_offset">;
1945def SVSTNT1W_SCATTER_INDEX_S : MInst<"svstnt1w_scatter[_{2}base]_index[_{d}]", "vPuld", "lUl", [IsScatterStore], MemEltTyInt32, "aarch64_sve_stnt1_scatter_scalar_offset">;
1946}
1947
1948////////////////////////////////////////////////////////////////////////////////
1949// SVE2 - Polynomial arithmetic
1950
1951let ArchGuard = "defined(__ARM_FEATURE_SVE2)" in {
1952def SVEORBT : SInst<"sveorbt[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eorbt">;
1953def SVEORBT_N : SInst<"sveorbt[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_eorbt">;
1954def SVEORTB : SInst<"sveortb[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eortb">;
1955def SVEORTB_N : SInst<"sveortb[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_eortb">;
1956def SVPMUL : SInst<"svpmul[_{d}]", "ddd", "Uc", MergeNone, "aarch64_sve_pmul">;
1957def SVPMUL_N : SInst<"svpmul[_n_{d}]", "dda", "Uc", MergeNone, "aarch64_sve_pmul">;
1958def SVPMULLB : SInst<"svpmullb[_{d}]", "dhh", "UsUl", MergeNone>;
1959def SVPMULLB_N : SInst<"svpmullb[_n_{d}]", "dhR", "UsUl", MergeNone>;
1960def SVPMULLB_PAIR : SInst<"svpmullb_pair[_{d}]", "ddd", "UcUi", MergeNone, "aarch64_sve_pmullb_pair">;
1961def SVPMULLB_PAIR_N : SInst<"svpmullb_pair[_n_{d}]", "dda", "UcUi", MergeNone, "aarch64_sve_pmullb_pair">;
1962def SVPMULLT : SInst<"svpmullt[_{d}]", "dhh", "UsUl", MergeNone>;
1963def SVPMULLT_N : SInst<"svpmullt[_n_{d}]", "dhR", "UsUl", MergeNone>;
1964def SVPMULLT_PAIR : SInst<"svpmullt_pair[_{d}]", "ddd", "UcUi", MergeNone, "aarch64_sve_pmullt_pair">;
1965def SVPMULLT_PAIR_N : SInst<"svpmullt_pair[_n_{d}]", "dda", "UcUi", MergeNone, "aarch64_sve_pmullt_pair">;
1966}
1967
1968////////////////////////////////////////////////////////////////////////////////
1969// SVE2 - Complex integer dot product
1970
1971let ArchGuard = "defined(__ARM_FEATURE_SVE2)" in {
1972def SVCDOT : SInst<"svcdot[_{d}]", "ddqqi", "il", MergeNone, "aarch64_sve_cdot", [], [ImmCheck<3, ImmCheckComplexRotAll90>]>;
1973def SVCDOT_LANE : SInst<"svcdot_lane[_{d}]", "ddqqii", "il", MergeNone, "aarch64_sve_cdot_lane", [], [ImmCheck<4, ImmCheckComplexRotAll90>,
1974 ImmCheck<3, ImmCheckLaneIndexDot, 2>]>;
1975}
1976
1977////////////////////////////////////////////////////////////////////////////////
1978// SVE2 - Floating-point widening multiply-accumulate
1979
1980let ArchGuard = "defined(__ARM_FEATURE_SVE2)" in {
1981def SVMLALB_F : SInst<"svmlalb[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlalb">;
1982def SVMLALB_F_N : SInst<"svmlalb[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlalb">;
1983def SVMLALB_F_LANE : SInst<"svmlalb_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlalb_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>;
1984def SVMLALT_F : SInst<"svmlalt[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlalt">;
1985def SVMLALT_F_N : SInst<"svmlalt[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlalt">;
1986def SVMLALT_F_LANE : SInst<"svmlalt_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlalt_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>;
1987def SVMLSLB_F : SInst<"svmlslb[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlslb">;
1988def SVMLSLB_F_N : SInst<"svmlslb[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlslb">;
1989def SVMLSLB_F_LANE : SInst<"svmlslb_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlslb_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>;
1990def SVMLSLT_F : SInst<"svmlslt[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlslt">;
1991def SVMLSLT_F_N : SInst<"svmlslt[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlslt">;
1992def SVMLSLT_F_LANE : SInst<"svmlslt_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlslt_lane", [], [ImmCheck<3, ImmCheckLaneIndex, 2>]>;
1993}
1994
1995////////////////////////////////////////////////////////////////////////////////
1996// SVE2 - Floating-point integer binary logarithm
1997
1998let ArchGuard = "defined(__ARM_FEATURE_SVE2)" in {
1999def SVLOGB_M : SInst<"svlogb[_{d}]", "xxPd", "hfd", MergeOp1, "aarch64_sve_flogb">;
2000def SVLOGB_X : SInst<"svlogb[_{d}]", "xPd", "hfd", MergeAnyExp, "aarch64_sve_flogb">;
2001def SVLOGB_Z : SInst<"svlogb[_{d}]", "xPd", "hfd", MergeZeroExp, "aarch64_sve_flogb">;
2002}
2003
2004////////////////////////////////////////////////////////////////////////////////
2005// SVE2 - Vector Histogram count
2006
2007let ArchGuard = "defined(__ARM_FEATURE_SVE2)" in {
2008def SVHISTCNT : SInst<"svhistcnt[_{d}]_z", "uPdd", "ilUiUl", MergeNone, "aarch64_sve_histcnt">;
2009def SVHISTSEG : SInst<"svhistseg[_{d}]", "udd", "cUc", MergeNone, "aarch64_sve_histseg">;
2010}
2011
2012////////////////////////////////////////////////////////////////////////////////
2013// SVE2 - Character match
2014
2015let ArchGuard = "defined(__ARM_FEATURE_SVE2)" in {
2016def SVMATCH : SInst<"svmatch[_{d}]", "PPdd", "csUcUs", MergeNone, "aarch64_sve_match">;
2017def SVNMATCH : SInst<"svnmatch[_{d}]", "PPdd", "csUcUs", MergeNone, "aarch64_sve_nmatch">;
2018}
2019
2020////////////////////////////////////////////////////////////////////////////////
2021// SVE2 - Contiguous conflict detection
2022let ArchGuard = "defined(__ARM_FEATURE_SVE2)" in {
2023def SVWHILERW_B : SInst<"svwhilerw[_{1}]", "Pcc", "cUc", MergeNone, "aarch64_sve_whilerw_b", [IsOverloadWhileRW]>;
2024def SVWHILERW_H : SInst<"svwhilerw[_{1}]", "Pcc", "sUsh", MergeNone, "aarch64_sve_whilerw_h", [IsOverloadWhileRW]>;
2025def SVWHILERW_S : SInst<"svwhilerw[_{1}]", "Pcc", "iUif", MergeNone, "aarch64_sve_whilerw_s", [IsOverloadWhileRW]>;
2026def SVWHILERW_D : SInst<"svwhilerw[_{1}]", "Pcc", "lUld", MergeNone, "aarch64_sve_whilerw_d", [IsOverloadWhileRW]>;
2027
2028def SVWHILEWR_B : SInst<"svwhilewr[_{1}]", "Pcc", "cUc", MergeNone, "aarch64_sve_whilewr_b", [IsOverloadWhileRW]>;
2029def SVWHILEWR_H : SInst<"svwhilewr[_{1}]", "Pcc", "sUsh", MergeNone, "aarch64_sve_whilewr_h", [IsOverloadWhileRW]>;
2030def SVWHILEWR_S : SInst<"svwhilewr[_{1}]", "Pcc", "iUif", MergeNone, "aarch64_sve_whilewr_s", [IsOverloadWhileRW]>;
2031def SVWHILEWR_D : SInst<"svwhilewr[_{1}]", "Pcc", "lUld", MergeNone, "aarch64_sve_whilewr_d", [IsOverloadWhileRW]>;
2032}
2033
2034let ArchGuard = "defined(__ARM_FEATURE_SVE2) && defined(__ARM_FEATURE_BF16_SCALAR_ARITHMETIC)" in {
2035def SVWHILERW_H_BF16 : SInst<"svwhilerw[_{1}]", "Pcc", "b", MergeNone, "aarch64_sve_whilerw_h", [IsOverloadWhileRW]>;
2036def SVWHILEWR_H_BF16 : SInst<"svwhilewr[_{1}]", "Pcc", "b", MergeNone, "aarch64_sve_whilewr_h", [IsOverloadWhileRW]>;
2037}
2038
2039////////////////////////////////////////////////////////////////////////////////
2040// SVE2 - Extended table lookup/permute
2041let ArchGuard = "defined(__ARM_FEATURE_SVE2)" in {
2042def SVTBL2 : SInst<"svtbl2[_{d}]", "d2u", "csilUcUsUiUlhfd", MergeNone>;
2043def SVTBX : SInst<"svtbx[_{d}]", "dddu", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tbx">;
2044}
2045
2046let ArchGuard = "defined(__ARM_FEATURE_SVE2) && defined(__ARM_FEATURE_SVE_BF16)" in {
2047def SVTBL2_BF16 : SInst<"svtbl2[_{d}]", "d2u", "b", MergeNone>;
2048def SVTBX_BF16 : SInst<"svtbx[_{d}]", "dddu", "b", MergeNone, "aarch64_sve_tbx">;
2049}
2050
2051////////////////////////////////////////////////////////////////////////////////
2052// SVE2 - Optional
2053
2054let ArchGuard = "defined(__ARM_FEATURE_SVE2_AES)" in {
2055def SVAESD : SInst<"svaesd[_{d}]", "ddd", "Uc", MergeNone, "aarch64_sve_aesd", [IsOverloadNone]>;
2056def SVAESIMC : SInst<"svaesimc[_{d}]", "dd", "Uc", MergeNone, "aarch64_sve_aesimc", [IsOverloadNone]>;
2057def SVAESE : SInst<"svaese[_{d}]", "ddd", "Uc", MergeNone, "aarch64_sve_aese", [IsOverloadNone]>;
2058def SVAESMC : SInst<"svaesmc[_{d}]", "dd", "Uc", MergeNone, "aarch64_sve_aesmc", [IsOverloadNone]>;
2059
2060def SVPMULLB_PAIR_U64 : SInst<"svpmullb_pair[_{d}]", "ddd", "Ul", MergeNone, "aarch64_sve_pmullb_pair">;
2061def SVPMULLB_PAIR_N_U64 : SInst<"svpmullb_pair[_n_{d}]", "dda", "Ul", MergeNone, "aarch64_sve_pmullb_pair">;
2062
2063def SVPMULLT_PAIR_U64 : SInst<"svpmullt_pair[_{d}]", "ddd", "Ul", MergeNone, "aarch64_sve_pmullt_pair">;
2064def SVPMULLT_PAIR_N_U64 : SInst<"svpmullt_pair[_n_{d}]", "dda", "Ul", MergeNone, "aarch64_sve_pmullt_pair">;
2065}
2066
2067let ArchGuard = "defined(__ARM_FEATURE_SVE2_SHA3)" in {
2068def SVRAX1 : SInst<"svrax1[_{d}]", "ddd", "lUl", MergeNone, "aarch64_sve_rax1", [IsOverloadNone]>;
2069}
2070
2071let ArchGuard = "defined(__ARM_FEATURE_SVE2_SM4)" in {
2072def SVSM4E : SInst<"svsm4e[_{d}]", "ddd", "Ui", MergeNone, "aarch64_sve_sm4e", [IsOverloadNone]>;
2073def SVSM4EKEY : SInst<"svsm4ekey[_{d}]", "ddd", "Ui", MergeNone, "aarch64_sve_sm4ekey", [IsOverloadNone]>;
2074}
2075
2076let ArchGuard = "defined (__ARM_FEATURE_SVE2_BITPERM)" in {
2077def SVBDEP : SInst<"svbdep[_{d}]", "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_bdep_x">;
2078def SVBDEP_N : SInst<"svbdep[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_bdep_x">;
2079def SVBEXT : SInst<"svbext[_{d}]", "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_bext_x">;
2080def SVBEXT_N : SInst<"svbext[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_bext_x">;
2081def SVBGRP : SInst<"svbgrp[_{d}]", "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_bgrp_x">;
2082def SVBGRP_N : SInst<"svbgrp[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_bgrp_x">;
2083}
2084