1//===--- arm_neon.td - ARM NEON compiler interface ------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the TableGen definitions from which the ARM NEON header
10// file will be generated. See ARM document DUI0348B.
11//
12//===----------------------------------------------------------------------===//
13
14include "arm_neon_incl.td"
15
16def OP_ADD : Op<(op "+", $p0, $p1)>;
17def OP_ADDL : Op<(op "+", (call "vmovl", $p0), (call "vmovl", $p1))>;
18def OP_ADDLHi : Op<(op "+", (call "vmovl_high", $p0),
19 (call "vmovl_high", $p1))>;
20def OP_ADDW : Op<(op "+", $p0, (call "vmovl", $p1))>;
21def OP_ADDWHi : Op<(op "+", $p0, (call "vmovl_high", $p1))>;
22def OP_SUB : Op<(op "-", $p0, $p1)>;
23def OP_SUBL : Op<(op "-", (call "vmovl", $p0), (call "vmovl", $p1))>;
24def OP_SUBLHi : Op<(op "-", (call "vmovl_high", $p0),
25 (call "vmovl_high", $p1))>;
26def OP_SUBW : Op<(op "-", $p0, (call "vmovl", $p1))>;
27def OP_SUBWHi : Op<(op "-", $p0, (call "vmovl_high", $p1))>;
28def OP_MUL : Op<(op "*", $p0, $p1)>;
29def OP_MLA : Op<(op "+", $p0, (op "*", $p1, $p2))>;
30def OP_MLAL : Op<(op "+", $p0, (call "vmull", $p1, $p2))>;
31def OP_MULLHi : Op<(call "vmull", (call "vget_high", $p0),
32 (call "vget_high", $p1))>;
33def OP_MULLHi_P64 : Op<(call "vmull",
34 (cast "poly64_t", (call "vget_high", $p0)),
35 (cast "poly64_t", (call "vget_high", $p1)))>;
36def OP_MULLHi_N : Op<(call "vmull_n", (call "vget_high", $p0), $p1)>;
37def OP_MLALHi : Op<(call "vmlal", $p0, (call "vget_high", $p1),
38 (call "vget_high", $p2))>;
39def OP_MLALHi_N : Op<(call "vmlal_n", $p0, (call "vget_high", $p1), $p2)>;
40def OP_MLS : Op<(op "-", $p0, (op "*", $p1, $p2))>;
41def OP_FMLS : Op<(call "vfma", $p0, (op "-", $p1), $p2)>;
42def OP_MLSL : Op<(op "-", $p0, (call "vmull", $p1, $p2))>;
43def OP_MLSLHi : Op<(call "vmlsl", $p0, (call "vget_high", $p1),
44 (call "vget_high", $p2))>;
45def OP_MLSLHi_N : Op<(call "vmlsl_n", $p0, (call "vget_high", $p1), $p2)>;
46def OP_MUL_N : Op<(op "*", $p0, (dup $p1))>;
47def OP_MULX_N : Op<(call "vmulx", $p0, (dup $p1))>;
48def OP_MLA_N : Op<(op "+", $p0, (op "*", $p1, (dup $p2)))>;
49def OP_MLS_N : Op<(op "-", $p0, (op "*", $p1, (dup $p2)))>;
50def OP_FMLA_N : Op<(call "vfma", $p0, $p1, (dup $p2))>;
51def OP_FMLS_N : Op<(call "vfma", $p0, (op "-", $p1), (dup $p2))>;
52def OP_MLAL_N : Op<(op "+", $p0, (call "vmull", $p1, (dup $p2)))>;
53def OP_MLSL_N : Op<(op "-", $p0, (call "vmull", $p1, (dup $p2)))>;
54def OP_MUL_LN : Op<(op "*", $p0, (call_mangled "splat_lane", $p1, $p2))>;
55def OP_MULX_LN : Op<(call "vmulx", $p0, (call_mangled "splat_lane", $p1, $p2))>;
56def OP_MULL_N : Op<(call "vmull", $p0, (dup $p1))>;
57def OP_MULL_LN : Op<(call "vmull", $p0, (call_mangled "splat_lane", $p1, $p2))>;
58def OP_MULLHi_LN: Op<(call "vmull", (call "vget_high", $p0), (call_mangled "splat_lane", $p1, $p2))>;
59def OP_MLA_LN : Op<(op "+", $p0, (op "*", $p1, (call_mangled "splat_lane", $p2, $p3)))>;
60def OP_MLS_LN : Op<(op "-", $p0, (op "*", $p1, (call_mangled "splat_lane", $p2, $p3)))>;
61def OP_MLAL_LN : Op<(op "+", $p0, (call "vmull", $p1, (call_mangled "splat_lane", $p2, $p3)))>;
62def OP_MLALHi_LN: Op<(op "+", $p0, (call "vmull", (call "vget_high", $p1),
63 (call_mangled "splat_lane", $p2, $p3)))>;
64def OP_MLSL_LN : Op<(op "-", $p0, (call "vmull", $p1, (call_mangled "splat_lane", $p2, $p3)))>;
65def OP_MLSLHi_LN : Op<(op "-", $p0, (call "vmull", (call "vget_high", $p1),
66 (call_mangled "splat_lane", $p2, $p3)))>;
67def OP_QDMULL_N : Op<(call "vqdmull", $p0, (dup $p1))>;
68def OP_QDMULL_LN : Op<(call "vqdmull", $p0, (call_mangled "splat_lane", $p1, $p2))>;
69def OP_QDMULLHi_LN : Op<(call "vqdmull", (call "vget_high", $p0),
70 (call_mangled "splat_lane", $p1, $p2))>;
71def OP_QDMLAL_N : Op<(call "vqdmlal", $p0, $p1, (dup $p2))>;
72def OP_QDMLAL_LN : Op<(call "vqdmlal", $p0, $p1, (call_mangled "splat_lane", $p2, $p3))>;
73def OP_QDMLALHi_LN : Op<(call "vqdmlal", $p0, (call "vget_high", $p1),
74 (call_mangled "splat_lane", $p2, $p3))>;
75def OP_QDMLSL_N : Op<(call "vqdmlsl", $p0, $p1, (dup $p2))>;
76def OP_QDMLSL_LN : Op<(call "vqdmlsl", $p0, $p1, (call_mangled "splat_lane", $p2, $p3))>;
77def OP_QDMLSLHi_LN : Op<(call "vqdmlsl", $p0, (call "vget_high", $p1),
78 (call_mangled "splat_lane", $p2, $p3))>;
79def OP_QDMULH_N : Op<(call "vqdmulh", $p0, (dup $p1))>;
80def OP_QDMULH_LN : Op<(call "vqdmulh", $p0, (call_mangled "splat_lane", $p1, $p2))>;
81def OP_QRDMULH_LN : Op<(call "vqrdmulh", $p0, (call_mangled "splat_lane", $p1, $p2))>;
82def OP_QRDMULH_N : Op<(call "vqrdmulh", $p0, (dup $p1))>;
83def OP_QRDMLAH : Op<(call "vqadd", $p0, (call "vqrdmulh", $p1, $p2))>;
84def OP_QRDMLSH : Op<(call "vqsub", $p0, (call "vqrdmulh", $p1, $p2))>;
85def OP_QRDMLAH_LN : Op<(call "vqadd", $p0, (call "vqrdmulh", $p1, (call_mangled "splat_lane", $p2, $p3)))>;
86def OP_QRDMLSH_LN : Op<(call "vqsub", $p0, (call "vqrdmulh", $p1, (call_mangled "splat_lane", $p2, $p3)))>;
87def OP_FMS_LN : Op<(call "vfma_lane", $p0, (op "-", $p1), $p2, $p3)>;
88def OP_FMS_LNQ : Op<(call "vfma_laneq", $p0, (op "-", $p1), $p2, $p3)>;
89def OP_TRN1 : Op<(shuffle $p0, $p1, (interleave (decimate mask0, 2),
90 (decimate mask1, 2)))>;
91def OP_ZIP1 : Op<(shuffle $p0, $p1, (lowhalf (interleave mask0, mask1)))>;
92def OP_UZP1 : Op<(shuffle $p0, $p1, (add (decimate mask0, 2),
93 (decimate mask1, 2)))>;
94def OP_TRN2 : Op<(shuffle $p0, $p1, (interleave
95 (decimate (rotl mask0, 1), 2),
96 (decimate (rotl mask1, 1), 2)))>;
97def OP_ZIP2 : Op<(shuffle $p0, $p1, (highhalf (interleave mask0, mask1)))>;
98def OP_UZP2 : Op<(shuffle $p0, $p1, (add (decimate (rotl mask0, 1), 2),
99 (decimate (rotl mask1, 1), 2)))>;
100def OP_EQ : Op<(cast "R", (op "==", $p0, $p1))>;
101def OP_GE : Op<(cast "R", (op ">=", $p0, $p1))>;
102def OP_LE : Op<(cast "R", (op "<=", $p0, $p1))>;
103def OP_GT : Op<(cast "R", (op ">", $p0, $p1))>;
104def OP_LT : Op<(cast "R", (op "<", $p0, $p1))>;
105def OP_NEG : Op<(op "-", $p0)>;
106def OP_NOT : Op<(op "~", $p0)>;
107def OP_AND : Op<(op "&", $p0, $p1)>;
108def OP_OR : Op<(op "|", $p0, $p1)>;
109def OP_XOR : Op<(op "^", $p0, $p1)>;
110def OP_ANDN : Op<(op "&", $p0, (op "~", $p1))>;
111def OP_ORN : Op<(op "|", $p0, (op "~", $p1))>;
112def OP_CAST : LOp<[(save_temp $promote, $p0),
113 (cast "R", $promote)]>;
114def OP_HI : Op<(shuffle $p0, $p0, (highhalf mask0))>;
115def OP_LO : Op<(shuffle $p0, $p0, (lowhalf mask0))>;
116def OP_CONC : Op<(shuffle $p0, $p1, (add mask0, mask1))>;
117def OP_DUP : Op<(dup $p0)>;
118def OP_DUP_LN : Op<(call_mangled "splat_lane", $p0, $p1)>;
119def OP_SEL : Op<(cast "R", (op "|",
120 (op "&", $p0, (cast $p0, $p1)),
121 (op "&", (op "~", $p0), (cast $p0, $p2))))>;
122def OP_REV16 : Op<(shuffle $p0, $p0, (rev 16, mask0))>;
123def OP_REV32 : Op<(shuffle $p0, $p0, (rev 32, mask0))>;
124def OP_REV64 : Op<(shuffle $p0, $p0, (rev 64, mask0))>;
125def OP_XTN : Op<(call "vcombine", $p0, (call "vmovn", $p1))>;
126def OP_SQXTUN : Op<(call "vcombine", (cast $p0, "U", $p0),
127 (call "vqmovun", $p1))>;
128def OP_QXTN : Op<(call "vcombine", $p0, (call "vqmovn", $p1))>;
129def OP_VCVT_NA_HI_F16 : Op<(call "vcombine", $p0, (call "vcvt_f16_f32", $p1))>;
130def OP_VCVT_NA_HI_F32 : Op<(call "vcombine", $p0, (call "vcvt_f32_f64", $p1))>;
131def OP_VCVT_EX_HI_F32 : Op<(call "vcvt_f32_f16", (call "vget_high", $p0))>;
132def OP_VCVT_EX_HI_F64 : Op<(call "vcvt_f64_f32", (call "vget_high", $p0))>;
133def OP_VCVTX_HI : Op<(call "vcombine", $p0, (call "vcvtx_f32", $p1))>;
134def OP_REINT : Op<(cast "R", $p0)>;
135def OP_ADDHNHi : Op<(call "vcombine", $p0, (call "vaddhn", $p1, $p2))>;
136def OP_RADDHNHi : Op<(call "vcombine", $p0, (call "vraddhn", $p1, $p2))>;
137def OP_SUBHNHi : Op<(call "vcombine", $p0, (call "vsubhn", $p1, $p2))>;
138def OP_RSUBHNHi : Op<(call "vcombine", $p0, (call "vrsubhn", $p1, $p2))>;
139def OP_ABDL : Op<(cast "R", (call "vmovl", (cast $p0, "U",
140 (call "vabd", $p0, $p1))))>;
141def OP_ABDLHi : Op<(call "vabdl", (call "vget_high", $p0),
142 (call "vget_high", $p1))>;
143def OP_ABA : Op<(op "+", $p0, (call "vabd", $p1, $p2))>;
144def OP_ABAL : Op<(op "+", $p0, (call "vabdl", $p1, $p2))>;
145def OP_ABALHi : Op<(call "vabal", $p0, (call "vget_high", $p1),
146 (call "vget_high", $p2))>;
147def OP_QDMULLHi : Op<(call "vqdmull", (call "vget_high", $p0),
148 (call "vget_high", $p1))>;
149def OP_QDMULLHi_N : Op<(call "vqdmull_n", (call "vget_high", $p0), $p1)>;
150def OP_QDMLALHi : Op<(call "vqdmlal", $p0, (call "vget_high", $p1),
151 (call "vget_high", $p2))>;
152def OP_QDMLALHi_N : Op<(call "vqdmlal_n", $p0, (call "vget_high", $p1), $p2)>;
153def OP_QDMLSLHi : Op<(call "vqdmlsl", $p0, (call "vget_high", $p1),
154 (call "vget_high", $p2))>;
155def OP_QDMLSLHi_N : Op<(call "vqdmlsl_n", $p0, (call "vget_high", $p1), $p2)>;
156def OP_DIV : Op<(op "/", $p0, $p1)>;
157def OP_LONG_HI : Op<(cast "R", (call (name_replace "_high_", "_"),
158 (call "vget_high", $p0), $p1))>;
159def OP_NARROW_HI : Op<(cast "R", (call "vcombine",
160 (cast "R", "H", $p0),
161 (cast "R", "H",
162 (call (name_replace "_high_", "_"),
163 $p1, $p2))))>;
164def OP_MOVL_HI : LOp<[(save_temp $a1, (call "vget_high", $p0)),
165 (cast "R",
166 (call "vshll_n", $a1, (literal "int32_t", "0")))]>;
167def OP_COPY_LN : Op<(call "vset_lane", (call "vget_lane", $p2, $p3), $p0, $p1)>;
168def OP_SCALAR_MUL_LN : Op<(op "*", $p0, (call "vget_lane", $p1, $p2))>;
169def OP_SCALAR_MULX_LN : Op<(call "vmulx", $p0, (call "vget_lane", $p1, $p2))>;
170def OP_SCALAR_VMULX_LN : LOp<[(save_temp $x, (call "vget_lane", $p0,
171 (literal "int32_t", "0"))),
172 (save_temp $y, (call "vget_lane", $p1, $p2)),
173 (save_temp $z, (call "vmulx", $x, $y)),
174 (call "vset_lane", $z, $p0, $p2)]>;
175def OP_SCALAR_VMULX_LNQ : LOp<[(save_temp $x, (call "vget_lane", $p0,
176 (literal "int32_t", "0"))),
177 (save_temp $y, (call "vget_lane", $p1, $p2)),
178 (save_temp $z, (call "vmulx", $x, $y)),
179 (call "vset_lane", $z, $p0, (literal "int32_t",
180 "0"))]>;
181class ScalarMulOp<string opname> :
182 Op<(call opname, $p0, (call "vget_lane", $p1, $p2))>;
183
184def OP_SCALAR_QDMULL_LN : ScalarMulOp<"vqdmull">;
185def OP_SCALAR_QDMULH_LN : ScalarMulOp<"vqdmulh">;
186def OP_SCALAR_QRDMULH_LN : ScalarMulOp<"vqrdmulh">;
187
188def OP_SCALAR_QRDMLAH_LN : Op<(call "vqadd", $p0, (call "vqrdmulh", $p1,
189 (call "vget_lane", $p2, $p3)))>;
190def OP_SCALAR_QRDMLSH_LN : Op<(call "vqsub", $p0, (call "vqrdmulh", $p1,
191 (call "vget_lane", $p2, $p3)))>;
192
193def OP_SCALAR_HALF_GET_LN : Op<(bitcast "float16_t",
194 (call "vget_lane",
195 (bitcast "int16x4_t", $p0), $p1))>;
196def OP_SCALAR_HALF_GET_LNQ : Op<(bitcast "float16_t",
197 (call "vget_lane",
198 (bitcast "int16x8_t", $p0), $p1))>;
199def OP_SCALAR_HALF_SET_LN : Op<(bitcast "float16x4_t",
200 (call "vset_lane",
201 (bitcast "int16_t", $p0),
202 (bitcast "int16x4_t", $p1), $p2))>;
203def OP_SCALAR_HALF_SET_LNQ : Op<(bitcast "float16x8_t",
204 (call "vset_lane",
205 (bitcast "int16_t", $p0),
206 (bitcast "int16x8_t", $p1), $p2))>;
207
208def OP_DOT_LN
209 : Op<(call "vdot", $p0, $p1,
210 (bitcast $p1, (call_mangled "splat_lane", (bitcast "32", $p2), $p3)))>;
211def OP_DOT_LNQ
212 : Op<(call "vdot", $p0, $p1,
213 (bitcast $p1, (call_mangled "splat_lane", (bitcast "32", $p2), $p3)))>;
214
215def OP_FMLAL_LN : Op<(call "vfmlal_low", $p0, $p1,
216 (dup_typed $p1, (call "vget_lane", $p2, $p3)))>;
217def OP_FMLSL_LN : Op<(call "vfmlsl_low", $p0, $p1,
218 (dup_typed $p1, (call "vget_lane", $p2, $p3)))>;
219def OP_FMLAL_LN_Hi : Op<(call "vfmlal_high", $p0, $p1,
220 (dup_typed $p1, (call "vget_lane", $p2, $p3)))>;
221def OP_FMLSL_LN_Hi : Op<(call "vfmlsl_high", $p0, $p1,
222 (dup_typed $p1, (call "vget_lane", $p2, $p3)))>;
223
224def OP_USDOT_LN
225 : Op<(call "vusdot", $p0, $p1,
226 (cast "8", "S", (call_mangled "splat_lane", (bitcast "int32x2_t", $p2), $p3)))>;
227def OP_USDOT_LNQ
228 : Op<(call "vusdot", $p0, $p1,
229 (cast "8", "S", (call_mangled "splat_lane", (bitcast "int32x4_t", $p2), $p3)))>;
230
231// sudot splats the second vector and then calls vusdot
232def OP_SUDOT_LN
233 : Op<(call "vusdot", $p0,
234 (cast "8", "U", (call_mangled "splat_lane", (bitcast "int32x2_t", $p2), $p3)), $p1)>;
235def OP_SUDOT_LNQ
236 : Op<(call "vusdot", $p0,
237 (cast "8", "U", (call_mangled "splat_lane", (bitcast "int32x4_t", $p2), $p3)), $p1)>;
238
239def OP_BFDOT_LN
240 : Op<(call "vbfdot", $p0, $p1,
241 (bitcast $p1, (call_mangled "splat_lane", (bitcast "float32x2_t", $p2), $p3)))>;
242
243def OP_BFDOT_LNQ
244 : Op<(call "vbfdot", $p0, $p1,
245 (bitcast $p1, (call_mangled "splat_lane", (bitcast "float32x4_t", $p2), $p3)))>;
246
247def OP_BFMLALB_LN
248 : Op<(call "vbfmlalb", $p0, $p1,
249 (dup_typed $p1, (call "vget_lane", $p2, $p3)))>;
250
251def OP_BFMLALT_LN
252 : Op<(call "vbfmlalt", $p0, $p1,
253 (dup_typed $p1, (call "vget_lane", $p2, $p3)))>;
254
255def OP_VCVT_F32_BF16
256 : Op<(bitcast "R",
257 (call "vshll_n", (bitcast "int16x4_t", $p0),
258 (literal "int32_t", "16")))>;
259def OP_VCVT_F32_BF16_LO
260 : Op<(call "vcvt_f32_bf16", (call "vget_low", $p0))>;
261def OP_VCVT_F32_BF16_HI
262 : Op<(call "vcvt_f32_bf16", (call "vget_high", $p0))>;
263
264def OP_VCVT_BF16_F32_LO_A64
265 : Op<(call "__a64_vcvtq_low_bf16", $p0)>;
266def OP_VCVT_BF16_F32_A64
267 : Op<(call "vget_low", (call "__a64_vcvtq_low_bf16", $p0))>;
268
269def OP_VCVT_BF16_F32_A32
270 : Op<(call "__a32_vcvt_bf16", $p0)>;
271
272def OP_VCVT_BF16_F32_LO_A32
273 : Op<(call "vcombine", (cast "bfloat16x4_t", (literal "uint64_t", "0ULL")),
274 (call "__a32_vcvt_bf16", $p0))>;
275def OP_VCVT_BF16_F32_HI_A32
276 : Op<(call "vcombine", (call "__a32_vcvt_bf16", $p1),
277 (call "vget_low", $p0))>;
278
279def OP_CVT_F32_BF16
280 : Op<(bitcast "R", (op "<<", (bitcast "int32_t", $p0),
281 (literal "int32_t", "16")))>;
282
283//===----------------------------------------------------------------------===//
284// Auxiliary Instructions
285//===----------------------------------------------------------------------===//
286
287// Splat operation - performs a range-checked splat over a vector
288def SPLAT : WInst<"splat_lane", ".(!q)I",
289 "UcUsUicsilPcPsfQUcQUsQUiQcQsQiQPcQPsQflUlQlQUlhdQhQdPlQPl">;
290def SPLATQ : WInst<"splat_laneq", ".(!Q)I",
291 "UcUsUicsilPcPsfQUcQUsQUiQcQsQiQPcQPsQflUlQlQUlhdQhQdPlQPl"> {
292 let isLaneQ = 1;
293}
294let ArchGuard = "defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC)" in {
295 def SPLAT_BF : WInst<"splat_lane", ".(!q)I", "bQb">;
296 def SPLATQ_BF : WInst<"splat_laneq", ".(!Q)I", "bQb"> {
297 let isLaneQ = 1;
298 }
299}
300
301//===----------------------------------------------------------------------===//
302// Intrinsics
303//===----------------------------------------------------------------------===//
304
305////////////////////////////////////////////////////////////////////////////////
306// E.3.1 Addition
307def VADD : IOpInst<"vadd", "...",
308 "csilfUcUsUiUlQcQsQiQlQfQUcQUsQUiQUl", OP_ADD>;
309def VADDL : SOpInst<"vaddl", "(>Q)..", "csiUcUsUi", OP_ADDL>;
310def VADDW : SOpInst<"vaddw", "(>Q)(>Q).", "csiUcUsUi", OP_ADDW>;
311def VHADD : SInst<"vhadd", "...", "csiUcUsUiQcQsQiQUcQUsQUi">;
312def VRHADD : SInst<"vrhadd", "...", "csiUcUsUiQcQsQiQUcQUsQUi">;
313def VQADD : SInst<"vqadd", "...", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
314def VADDHN : IInst<"vaddhn", "<QQ", "silUsUiUl">;
315def VRADDHN : IInst<"vraddhn", "<QQ", "silUsUiUl">;
316
317////////////////////////////////////////////////////////////////////////////////
318// E.3.2 Multiplication
319def VMUL : IOpInst<"vmul", "...", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_MUL>;
320def VMULP : SInst<"vmul", "...", "PcQPc">;
321def VMLA : IOpInst<"vmla", "....", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_MLA>;
322def VMLAL : SOpInst<"vmlal", "(>Q)(>Q)..", "csiUcUsUi", OP_MLAL>;
323def VMLS : IOpInst<"vmls", "....", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_MLS>;
324def VMLSL : SOpInst<"vmlsl", "(>Q)(>Q)..", "csiUcUsUi", OP_MLSL>;
325def VQDMULH : SInst<"vqdmulh", "...", "siQsQi">;
326def VQRDMULH : SInst<"vqrdmulh", "...", "siQsQi">;
327
328let ArchGuard = "defined(__ARM_FEATURE_QRDMX)" in {
329def VQRDMLAH : SOpInst<"vqrdmlah", "....", "siQsQi", OP_QRDMLAH>;
330def VQRDMLSH : SOpInst<"vqrdmlsh", "....", "siQsQi", OP_QRDMLSH>;
331}
332
333def VQDMLAL : SInst<"vqdmlal", "(>Q)(>Q)..", "si">;
334def VQDMLSL : SInst<"vqdmlsl", "(>Q)(>Q)..", "si">;
335def VMULL : SInst<"vmull", "(>Q)..", "csiUcUsUiPc">;
336def VQDMULL : SInst<"vqdmull", "(>Q)..", "si">;
337
338////////////////////////////////////////////////////////////////////////////////
339// E.3.3 Subtraction
340def VSUB : IOpInst<"vsub", "...",
341 "csilfUcUsUiUlQcQsQiQlQfQUcQUsQUiQUl", OP_SUB>;
342def VSUBL : SOpInst<"vsubl", "(>Q)..", "csiUcUsUi", OP_SUBL>;
343def VSUBW : SOpInst<"vsubw", "(>Q)(>Q).", "csiUcUsUi", OP_SUBW>;
344def VQSUB : SInst<"vqsub", "...", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
345def VHSUB : SInst<"vhsub", "...", "csiUcUsUiQcQsQiQUcQUsQUi">;
346def VSUBHN : IInst<"vsubhn", "<QQ", "silUsUiUl">;
347def VRSUBHN : IInst<"vrsubhn", "<QQ", "silUsUiUl">;
348
349////////////////////////////////////////////////////////////////////////////////
350// E.3.4 Comparison
351def VCEQ : IOpInst<"vceq", "U..", "csifUcUsUiPcQcQsQiQfQUcQUsQUiQPc", OP_EQ>;
352def VCGE : SOpInst<"vcge", "U..", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_GE>;
353let InstName = "vcge" in
354def VCLE : SOpInst<"vcle", "U..", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_LE>;
355def VCGT : SOpInst<"vcgt", "U..", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_GT>;
356let InstName = "vcgt" in
357def VCLT : SOpInst<"vclt", "U..", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_LT>;
358let InstName = "vacge" in {
359def VCAGE : IInst<"vcage", "U..", "fQf">;
360def VCALE : IInst<"vcale", "U..", "fQf">;
361}
362let InstName = "vacgt" in {
363def VCAGT : IInst<"vcagt", "U..", "fQf">;
364def VCALT : IInst<"vcalt", "U..", "fQf">;
365}
366def VTST : WInst<"vtst", "U..", "csiUcUsUiPcPsQcQsQiQUcQUsQUiQPcQPs">;
367
368////////////////////////////////////////////////////////////////////////////////
369// E.3.5 Absolute Difference
370def VABD : SInst<"vabd", "...", "csiUcUsUifQcQsQiQUcQUsQUiQf">;
371def VABDL : SOpInst<"vabdl", "(>Q)..", "csiUcUsUi", OP_ABDL>;
372def VABA : SOpInst<"vaba", "....", "csiUcUsUiQcQsQiQUcQUsQUi", OP_ABA>;
373def VABAL : SOpInst<"vabal", "(>Q)(>Q)..", "csiUcUsUi", OP_ABAL>;
374
375////////////////////////////////////////////////////////////////////////////////
376// E.3.6 Max/Min
377def VMAX : SInst<"vmax", "...", "csiUcUsUifQcQsQiQUcQUsQUiQf">;
378def VMIN : SInst<"vmin", "...", "csiUcUsUifQcQsQiQUcQUsQUiQf">;
379
380////////////////////////////////////////////////////////////////////////////////
381// E.3.7 Pairwise Addition
382def VPADD : IInst<"vpadd", "...", "csiUcUsUif">;
383def VPADDL : SInst<"vpaddl", ">.", "csiUcUsUiQcQsQiQUcQUsQUi">;
384def VPADAL : SInst<"vpadal", ">>.", "csiUcUsUiQcQsQiQUcQUsQUi">;
385
386////////////////////////////////////////////////////////////////////////////////
387// E.3.8-9 Folding Max/Min
388def VPMAX : SInst<"vpmax", "...", "csiUcUsUif">;
389def VPMIN : SInst<"vpmin", "...", "csiUcUsUif">;
390
391////////////////////////////////////////////////////////////////////////////////
392// E.3.10 Reciprocal/Sqrt
393def VRECPS : IInst<"vrecps", "...", "fQf">;
394def VRSQRTS : IInst<"vrsqrts", "...", "fQf">;
395
396////////////////////////////////////////////////////////////////////////////////
397// E.3.11 Shifts by signed variable
398def VSHL : SInst<"vshl", "..S", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
399def VQSHL : SInst<"vqshl", "..S", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
400def VRSHL : SInst<"vrshl", "..S", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
401def VQRSHL : SInst<"vqrshl", "..S", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
402
403////////////////////////////////////////////////////////////////////////////////
404// E.3.12 Shifts by constant
405let isShift = 1 in {
406def VSHR_N : SInst<"vshr_n", "..I", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
407def VSHL_N : IInst<"vshl_n", "..I", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
408def VRSHR_N : SInst<"vrshr_n", "..I", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
409def VSRA_N : SInst<"vsra_n", "...I", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
410def VRSRA_N : SInst<"vrsra_n", "...I", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
411def VQSHL_N : SInst<"vqshl_n", "..I", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">;
412def VQSHLU_N : SInst<"vqshlu_n", "U.I", "csilQcQsQiQl">;
413def VSHRN_N : IInst<"vshrn_n", "<QI", "silUsUiUl">;
414def VQSHRUN_N : SInst<"vqshrun_n", "(<U)QI", "sil">;
415def VQRSHRUN_N : SInst<"vqrshrun_n", "(<U)QI", "sil">;
416def VQSHRN_N : SInst<"vqshrn_n", "<QI", "silUsUiUl">;
417def VRSHRN_N : IInst<"vrshrn_n", "<QI", "silUsUiUl">;
418def VQRSHRN_N : SInst<"vqrshrn_n", "<QI", "silUsUiUl">;
419def VSHLL_N : SInst<"vshll_n", "(>Q).I", "csiUcUsUi">;
420
421////////////////////////////////////////////////////////////////////////////////
422// E.3.13 Shifts with insert
423def VSRI_N : WInst<"vsri_n", "...I",
424 "csilUcUsUiUlPcPsQcQsQiQlQUcQUsQUiQUlQPcQPs">;
425def VSLI_N : WInst<"vsli_n", "...I",
426 "csilUcUsUiUlPcPsQcQsQiQlQUcQUsQUiQUlQPcQPs">;
427}
428
429////////////////////////////////////////////////////////////////////////////////
430// E.3.14 Loads and stores of a single vector
431def VLD1 : WInst<"vld1", ".(c*!)",
432 "QUcQUsQUiQUlQcQsQiQlQfQPcQPsUcUsUiUlcsilfPcPs">;
433def VLD1_X2 : WInst<"vld1_x2", "2(c*!)",
434 "cfilsUcUiUlUsQcQfQiQlQsQUcQUiQUlQUsPcPsQPcQPs">;
435def VLD1_X3 : WInst<"vld1_x3", "3(c*!)",
436 "cfilsUcUiUlUsQcQfQiQlQsQUcQUiQUlQUsPcPsQPcQPs">;
437def VLD1_X4 : WInst<"vld1_x4", "4(c*!)",
438 "cfilsUcUiUlUsQcQfQiQlQsQUcQUiQUlQUsPcPsQPcQPs">;
439def VLD1_LANE : WInst<"vld1_lane", ".(c*!).I",
440 "QUcQUsQUiQUlQcQsQiQlQfQPcQPsUcUsUiUlcsilfPcPs">;
441def VLD1_DUP : WInst<"vld1_dup", ".(c*!)",
442 "QUcQUsQUiQUlQcQsQiQlQfQPcQPsUcUsUiUlcsilfPcPs">;
443def VST1 : WInst<"vst1", "v*(.!)",
444 "QUcQUsQUiQUlQcQsQiQlQfQPcQPsUcUsUiUlcsilfPcPs">;
445def VST1_X2 : WInst<"vst1_x2", "v*(2!)",
446 "cfilsUcUiUlUsQcQfQiQlQsQUcQUiQUlQUsPcPsQPcQPs">;
447def VST1_X3 : WInst<"vst1_x3", "v*(3!)",
448 "cfilsUcUiUlUsQcQfQiQlQsQUcQUiQUlQUsPcPsQPcQPs">;
449def VST1_X4 : WInst<"vst1_x4", "v*(4!)",
450 "cfilsUcUiUlUsQcQfQiQlQsQUcQUiQUlQUsPcPsQPcQPs">;
451def VST1_LANE : WInst<"vst1_lane", "v*(.!)I",
452 "QUcQUsQUiQUlQcQsQiQlQfQPcQPsUcUsUiUlcsilfPcPs">;
453let ArchGuard = "(__ARM_FP & 2)" in {
454def VLD1_F16 : WInst<"vld1", ".(c*!)", "hQh">;
455def VLD1_X2_F16 : WInst<"vld1_x2", "2(c*!)", "hQh">;
456def VLD1_X3_F16 : WInst<"vld1_x3", "3(c*!)", "hQh">;
457def VLD1_X4_F16 : WInst<"vld1_x4", "4(c*!)", "hQh">;
458def VLD1_LANE_F16 : WInst<"vld1_lane", ".(c*!).I", "hQh">;
459def VLD1_DUP_F16 : WInst<"vld1_dup", ".(c*!)", "hQh">;
460def VST1_F16 : WInst<"vst1", "v*(.!)", "hQh">;
461def VST1_X2_F16 : WInst<"vst1_x2", "v*(2!)", "hQh">;
462def VST1_X3_F16 : WInst<"vst1_x3", "v*(3!)", "hQh">;
463def VST1_X4_F16 : WInst<"vst1_x4", "v*(4!)", "hQh">;
464def VST1_LANE_F16 : WInst<"vst1_lane", "v*(.!)I", "hQh">;
465}
466
467////////////////////////////////////////////////////////////////////////////////
468// E.3.15 Loads and stores of an N-element structure
469def VLD2 : WInst<"vld2", "2(c*!)", "QUcQUsQUiQcQsQiQfQPcQPsUcUsUiUlcsilfPcPs">;
470def VLD3 : WInst<"vld3", "3(c*!)", "QUcQUsQUiQcQsQiQfQPcQPsUcUsUiUlcsilfPcPs">;
471def VLD4 : WInst<"vld4", "4(c*!)", "QUcQUsQUiQcQsQiQfQPcQPsUcUsUiUlcsilfPcPs">;
472def VLD2_DUP : WInst<"vld2_dup", "2(c*!)",
473 "UcUsUiUlcsilfPcPsQcQfQiQlQsQPcQPsQUcQUiQUlQUs">;
474def VLD3_DUP : WInst<"vld3_dup", "3(c*!)",
475 "UcUsUiUlcsilfPcPsQcQfQiQlQsQPcQPsQUcQUiQUlQUs">;
476def VLD4_DUP : WInst<"vld4_dup", "4(c*!)",
477 "UcUsUiUlcsilfPcPsQcQfQiQlQsQPcQPsQUcQUiQUlQUs">;
478def VLD2_LANE : WInst<"vld2_lane", "2(c*!)2I", "QUsQUiQsQiQfQPsUcUsUicsifPcPs">;
479def VLD3_LANE : WInst<"vld3_lane", "3(c*!)3I", "QUsQUiQsQiQfQPsUcUsUicsifPcPs">;
480def VLD4_LANE : WInst<"vld4_lane", "4(c*!)4I", "QUsQUiQsQiQfQPsUcUsUicsifPcPs">;
481def VST2 : WInst<"vst2", "v*(2!)", "QUcQUsQUiQcQsQiQfQPcQPsUcUsUiUlcsilfPcPs">;
482def VST3 : WInst<"vst3", "v*(3!)", "QUcQUsQUiQcQsQiQfQPcQPsUcUsUiUlcsilfPcPs">;
483def VST4 : WInst<"vst4", "v*(4!)", "QUcQUsQUiQcQsQiQfQPcQPsUcUsUiUlcsilfPcPs">;
484def VST2_LANE : WInst<"vst2_lane", "v*(2!)I", "QUsQUiQsQiQfQPsUcUsUicsifPcPs">;
485def VST3_LANE : WInst<"vst3_lane", "v*(3!)I", "QUsQUiQsQiQfQPsUcUsUicsifPcPs">;
486def VST4_LANE : WInst<"vst4_lane", "v*(4!)I", "QUsQUiQsQiQfQPsUcUsUicsifPcPs">;
487let ArchGuard = "(__ARM_FP & 2)" in {
488def VLD2_F16 : WInst<"vld2", "2(c*!)", "hQh">;
489def VLD3_F16 : WInst<"vld3", "3(c*!)", "hQh">;
490def VLD4_F16 : WInst<"vld4", "4(c*!)", "hQh">;
491def VLD2_DUP_F16 : WInst<"vld2_dup", "2(c*!)", "hQh">;
492def VLD3_DUP_F16 : WInst<"vld3_dup", "3(c*!)", "hQh">;
493def VLD4_DUP_F16 : WInst<"vld4_dup", "4(c*!)", "hQh">;
494def VLD2_LANE_F16 : WInst<"vld2_lane", "2(c*!)2I", "hQh">;
495def VLD3_LANE_F16 : WInst<"vld3_lane", "3(c*!)3I", "hQh">;
496def VLD4_LANE_F16 : WInst<"vld4_lane", "4(c*!)4I", "hQh">;
497def VST2_F16 : WInst<"vst2", "v*(2!)", "hQh">;
498def VST3_F16 : WInst<"vst3", "v*(3!)", "hQh">;
499def VST4_F16 : WInst<"vst4", "v*(4!)", "hQh">;
500def VST2_LANE_F16 : WInst<"vst2_lane", "v*(2!)I", "hQh">;
501def VST3_LANE_F16 : WInst<"vst3_lane", "v*(3!)I", "hQh">;
502def VST4_LANE_F16 : WInst<"vst4_lane", "v*(4!)I", "hQh">;
503}
504
505////////////////////////////////////////////////////////////////////////////////
506// E.3.16 Extract lanes from a vector
507let InstName = "vmov" in
508def VGET_LANE : IInst<"vget_lane", "1.I",
509 "UcUsUicsiPcPsfQUcQUsQUiQcQsQiQPcQPsQflUlQlQUl">;
510
511////////////////////////////////////////////////////////////////////////////////
512// E.3.17 Set lanes within a vector
513let InstName = "vmov" in
514def VSET_LANE : IInst<"vset_lane", ".1.I",
515 "UcUsUicsiPcPsfQUcQUsQUiQcQsQiQPcQPsQflUlQlQUl">;
516
517////////////////////////////////////////////////////////////////////////////////
518// E.3.18 Initialize a vector from bit pattern
519def VCREATE : NoTestOpInst<"vcreate", ".(IU>)", "csihfUcUsUiUlPcPsl", OP_CAST> {
520 let BigEndianSafe = 1;
521}
522
523////////////////////////////////////////////////////////////////////////////////
524// E.3.19 Set all lanes to same value
525let InstName = "vmov" in {
526def VDUP_N : WOpInst<"vdup_n", ".1",
527 "UcUsUicsiPcPshfQUcQUsQUiQcQsQiQPcQPsQhQflUlQlQUl",
528 OP_DUP>;
529def VMOV_N : WOpInst<"vmov_n", ".1",
530 "UcUsUicsiPcPshfQUcQUsQUiQcQsQiQPcQPsQhQflUlQlQUl",
531 OP_DUP>;
532}
533let InstName = "" in
534def VDUP_LANE: WOpInst<"vdup_lane", ".qI",
535 "UcUsUicsiPcPsfQUcQUsQUiQcQsQiQPcQPsQflUlQlQUl",
536 OP_DUP_LN>;
537
538////////////////////////////////////////////////////////////////////////////////
539// E.3.20 Combining vectors
540def VCOMBINE : NoTestOpInst<"vcombine", "Q..", "csilhfUcUsUiUlPcPs", OP_CONC>;
541
542////////////////////////////////////////////////////////////////////////////////
543// E.3.21 Splitting vectors
544// Note that the ARM NEON Reference 2.0 mistakenly document the vget_high_f16()
545// and vget_low_f16() intrinsics as AArch64-only. We (and GCC) support all
546// versions of these intrinsics in both AArch32 and AArch64 architectures. See
547// D45668 for more details.
548let InstName = "vmov" in {
549def VGET_HIGH : NoTestOpInst<"vget_high", ".Q", "csilhfUcUsUiUlPcPs", OP_HI>;
550def VGET_LOW : NoTestOpInst<"vget_low", ".Q", "csilhfUcUsUiUlPcPs", OP_LO>;
551}
552
553////////////////////////////////////////////////////////////////////////////////
554// E.3.22 Converting vectors
555
556let ArchGuard = "(__ARM_FP & 2)" in {
557 def VCVT_F16_F32 : SInst<"vcvt_f16_f32", "(<q)(.!)", "Hf">;
558 def VCVT_F32_F16 : SInst<"vcvt_f32_f16", "(>Q)(.!)", "h">;
559}
560
561def VCVT_S32 : SInst<"vcvt_s32", "S.", "fQf">;
562def VCVT_U32 : SInst<"vcvt_u32", "U.", "fQf">;
563def VCVT_F32 : SInst<"vcvt_f32", "F(.!)", "iUiQiQUi">;
564let isVCVT_N = 1 in {
565def VCVT_N_S32 : SInst<"vcvt_n_s32", "S.I", "fQf">;
566def VCVT_N_U32 : SInst<"vcvt_n_u32", "U.I", "fQf">;
567def VCVT_N_F32 : SInst<"vcvt_n_f32", "F(.!)I", "iUiQiQUi">;
568}
569
570def VMOVN : IInst<"vmovn", "<Q", "silUsUiUl">;
571def VMOVL : SInst<"vmovl", "(>Q).", "csiUcUsUi">;
572def VQMOVN : SInst<"vqmovn", "<Q", "silUsUiUl">;
573def VQMOVUN : SInst<"vqmovun", "(<U)Q", "sil">;
574
575////////////////////////////////////////////////////////////////////////////////
576// E.3.23-24 Table lookup, Extended table lookup
577let InstName = "vtbl" in {
578def VTBL1 : WInst<"vtbl1", "..p", "UccPc">;
579def VTBL2 : WInst<"vtbl2", ".2p", "UccPc">;
580def VTBL3 : WInst<"vtbl3", ".3p", "UccPc">;
581def VTBL4 : WInst<"vtbl4", ".4p", "UccPc">;
582}
583let InstName = "vtbx" in {
584def VTBX1 : WInst<"vtbx1", "...p", "UccPc">;
585def VTBX2 : WInst<"vtbx2", "..2p", "UccPc">;
586def VTBX3 : WInst<"vtbx3", "..3p", "UccPc">;
587def VTBX4 : WInst<"vtbx4", "..4p", "UccPc">;
588}
589
590////////////////////////////////////////////////////////////////////////////////
591// E.3.25 Operations with a scalar value
592def VMLA_LANE : IOpInst<"vmla_lane", "...qI",
593 "siUsUifQsQiQUsQUiQf", OP_MLA_LN>;
594def VMLAL_LANE : SOpInst<"vmlal_lane", "(>Q)(>Q)..I", "siUsUi", OP_MLAL_LN>;
595def VQDMLAL_LANE : SOpInst<"vqdmlal_lane", "(>Q)(>Q)..I", "si", OP_QDMLAL_LN>;
596def VMLS_LANE : IOpInst<"vmls_lane", "...qI",
597 "siUsUifQsQiQUsQUiQf", OP_MLS_LN>;
598def VMLSL_LANE : SOpInst<"vmlsl_lane", "(>Q)(>Q)..I", "siUsUi", OP_MLSL_LN>;
599def VQDMLSL_LANE : SOpInst<"vqdmlsl_lane", "(>Q)(>Q)..I", "si", OP_QDMLSL_LN>;
600def VMUL_N : IOpInst<"vmul_n", "..1", "sifUsUiQsQiQfQUsQUi", OP_MUL_N>;
601def VMUL_LANE : IOpInst<"vmul_lane", "..qI",
602 "sifUsUiQsQiQfQUsQUi", OP_MUL_LN>;
603def VMULL_N : SOpInst<"vmull_n", "(>Q).1", "siUsUi", OP_MULL_N>;
604def VMULL_LANE : SOpInst<"vmull_lane", "(>Q)..I", "siUsUi", OP_MULL_LN>;
605def VQDMULL_N : SOpInst<"vqdmull_n", "(>Q).1", "si", OP_QDMULL_N>;
606def VQDMULL_LANE : SOpInst<"vqdmull_lane", "(>Q)..I", "si", OP_QDMULL_LN>;
607def VQDMULH_N : SOpInst<"vqdmulh_n", "..1", "siQsQi", OP_QDMULH_N>;
608def VQRDMULH_N : SOpInst<"vqrdmulh_n", "..1", "siQsQi", OP_QRDMULH_N>;
609
610let ArchGuard = "!defined(__aarch64__)" in {
611def VQDMULH_LANE : SOpInst<"vqdmulh_lane", "..qI", "siQsQi", OP_QDMULH_LN>;
612def VQRDMULH_LANE : SOpInst<"vqrdmulh_lane", "..qI", "siQsQi", OP_QRDMULH_LN>;
613}
614let ArchGuard = "defined(__aarch64__)" in {
615def A64_VQDMULH_LANE : SInst<"vqdmulh_lane", "..(!q)I", "siQsQi">;
616def A64_VQRDMULH_LANE : SInst<"vqrdmulh_lane", "..(!q)I", "siQsQi">;
617}
618
619let ArchGuard = "defined(__ARM_FEATURE_QRDMX)" in {
620def VQRDMLAH_LANE : SOpInst<"vqrdmlah_lane", "...qI", "siQsQi", OP_QRDMLAH_LN>;
621def VQRDMLSH_LANE : SOpInst<"vqrdmlsh_lane", "...qI", "siQsQi", OP_QRDMLSH_LN>;
622}
623
624def VMLA_N : IOpInst<"vmla_n", "...1", "siUsUifQsQiQUsQUiQf", OP_MLA_N>;
625def VMLAL_N : SOpInst<"vmlal_n", "(>Q)(>Q).1", "siUsUi", OP_MLAL_N>;
626def VQDMLAL_N : SOpInst<"vqdmlal_n", "(>Q)(>Q).1", "si", OP_QDMLAL_N>;
627def VMLS_N : IOpInst<"vmls_n", "...1", "siUsUifQsQiQUsQUiQf", OP_MLS_N>;
628def VMLSL_N : SOpInst<"vmlsl_n", "(>Q)(>Q).1", "siUsUi", OP_MLSL_N>;
629def VQDMLSL_N : SOpInst<"vqdmlsl_n", "(>Q)(>Q).1", "si", OP_QDMLSL_N>;
630
631////////////////////////////////////////////////////////////////////////////////
632// E.3.26 Vector Extract
633def VEXT : WInst<"vext", "...I",
634 "cUcPcsUsPsiUilUlfQcQUcQPcQsQUsQPsQiQUiQlQUlQf">;
635
636////////////////////////////////////////////////////////////////////////////////
637// E.3.27 Reverse vector elements
638def VREV64 : WOpInst<"vrev64", "..", "csiUcUsUiPcPsfQcQsQiQUcQUsQUiQPcQPsQf",
639 OP_REV64>;
640def VREV32 : WOpInst<"vrev32", "..", "csUcUsPcPsQcQsQUcQUsQPcQPs", OP_REV32>;
641def VREV16 : WOpInst<"vrev16", "..", "cUcPcQcQUcQPc", OP_REV16>;
642
643////////////////////////////////////////////////////////////////////////////////
644// E.3.28 Other single operand arithmetic
645def VABS : SInst<"vabs", "..", "csifQcQsQiQf">;
646def VQABS : SInst<"vqabs", "..", "csiQcQsQi">;
647def VNEG : SOpInst<"vneg", "..", "csifQcQsQiQf", OP_NEG>;
648def VQNEG : SInst<"vqneg", "..", "csiQcQsQi">;
649def VCLS : SInst<"vcls", "S.", "csiUcUsUiQcQsQiQUcQUsQUi">;
650def VCLZ : IInst<"vclz", "..", "csiUcUsUiQcQsQiQUcQUsQUi">;
651def VCNT : WInst<"vcnt", "..", "UccPcQUcQcQPc">;
652def VRECPE : SInst<"vrecpe", "..", "fUiQfQUi">;
653def VRSQRTE : SInst<"vrsqrte", "..", "fUiQfQUi">;
654
655////////////////////////////////////////////////////////////////////////////////
656// E.3.29 Logical operations
657def VMVN : LOpInst<"vmvn", "..", "csiUcUsUiPcQcQsQiQUcQUsQUiQPc", OP_NOT>;
658def VAND : LOpInst<"vand", "...", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl", OP_AND>;
659def VORR : LOpInst<"vorr", "...", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl", OP_OR>;
660def VEOR : LOpInst<"veor", "...", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl", OP_XOR>;
661def VBIC : LOpInst<"vbic", "...", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl", OP_ANDN>;
662def VORN : LOpInst<"vorn", "...", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl", OP_ORN>;
663let isHiddenLInst = 1 in
664def VBSL : SInst<"vbsl", ".U..",
665 "csilUcUsUiUlfPcPsQcQsQiQlQUcQUsQUiQUlQfQPcQPs">;
666
667////////////////////////////////////////////////////////////////////////////////
668// E.3.30 Transposition operations
669def VTRN : WInst<"vtrn", "2..", "csiUcUsUifPcPsQcQsQiQUcQUsQUiQfQPcQPs">;
670def VZIP : WInst<"vzip", "2..", "csiUcUsUifPcPsQcQsQiQUcQUsQUiQfQPcQPs">;
671def VUZP : WInst<"vuzp", "2..", "csiUcUsUifPcPsQcQsQiQUcQUsQUiQfQPcQPs">;
672
673////////////////////////////////////////////////////////////////////////////////
674
675class REINTERPRET_CROSS_SELF<string Types> :
676 NoTestOpInst<"vreinterpret", "..", Types, OP_REINT> {
677 let CartesianProductWith = Types;
678}
679
680multiclass REINTERPRET_CROSS_TYPES<string TypesA, string TypesB> {
681 def AXB: NoTestOpInst<"vreinterpret", "..", TypesA, OP_REINT> {
682 let CartesianProductWith = TypesB;
683 }
684 def BXA: NoTestOpInst<"vreinterpret", "..", TypesB, OP_REINT> {
685 let CartesianProductWith = TypesA;
686 }
687}
688
689// E.3.31 Vector reinterpret cast operations
690def VREINTERPRET : REINTERPRET_CROSS_SELF<"csilUcUsUiUlhfPcPsQcQsQiQlQUcQUsQUiQUlQhQfQPcQPs"> {
691 let ArchGuard = "!defined(__aarch64__)";
692 let BigEndianSafe = 1;
693}
694
695////////////////////////////////////////////////////////////////////////////////
696// Vector fused multiply-add operations
697
698let ArchGuard = "defined(__ARM_FEATURE_FMA)" in {
699 def VFMA : SInst<"vfma", "....", "fQf">;
700 def VFMS : SOpInst<"vfms", "....", "fQf", OP_FMLS>;
701 def FMLA_N_F32 : SOpInst<"vfma_n", "...1", "fQf", OP_FMLA_N>;
702}
703
704////////////////////////////////////////////////////////////////////////////////
705// fp16 vector operations
706def SCALAR_HALF_GET_LANE : IOpInst<"vget_lane", "1.I", "h", OP_SCALAR_HALF_GET_LN>;
707def SCALAR_HALF_SET_LANE : IOpInst<"vset_lane", ".1.I", "h", OP_SCALAR_HALF_SET_LN>;
708def SCALAR_HALF_GET_LANEQ : IOpInst<"vget_lane", "1.I", "Qh", OP_SCALAR_HALF_GET_LNQ>;
709def SCALAR_HALF_SET_LANEQ : IOpInst<"vset_lane", ".1.I", "Qh", OP_SCALAR_HALF_SET_LNQ>;
710
711////////////////////////////////////////////////////////////////////////////////
712// Non poly128_t vaddp for Arm and AArch64
713// TODO: poly128_t not implemented on arm32
714def VADDP : WInst<"vadd", "...", "PcPsPlQPcQPsQPl">;
715
716////////////////////////////////////////////////////////////////////////////////
717// AArch64 Intrinsics
718
719let ArchGuard = "defined(__aarch64__)" in {
720
721////////////////////////////////////////////////////////////////////////////////
722// Load/Store
723def LD1 : WInst<"vld1", ".(c*!)", "dQdPlQPl">;
724def LD2 : WInst<"vld2", "2(c*!)", "QUlQldQdPlQPl">;
725def LD3 : WInst<"vld3", "3(c*!)", "QUlQldQdPlQPl">;
726def LD4 : WInst<"vld4", "4(c*!)", "QUlQldQdPlQPl">;
727def ST1 : WInst<"vst1", "v*(.!)", "dQdPlQPl">;
728def ST2 : WInst<"vst2", "v*(2!)", "QUlQldQdPlQPl">;
729def ST3 : WInst<"vst3", "v*(3!)", "QUlQldQdPlQPl">;
730def ST4 : WInst<"vst4", "v*(4!)", "QUlQldQdPlQPl">;
731
732def LD1_X2 : WInst<"vld1_x2", "2(c*!)",
733 "dQdPlQPl">;
734def LD1_X3 : WInst<"vld1_x3", "3(c*!)",
735 "dQdPlQPl">;
736def LD1_X4 : WInst<"vld1_x4", "4(c*!)",
737 "dQdPlQPl">;
738
739def ST1_X2 : WInst<"vst1_x2", "v*(2!)", "dQdPlQPl">;
740def ST1_X3 : WInst<"vst1_x3", "v*(3!)", "dQdPlQPl">;
741def ST1_X4 : WInst<"vst1_x4", "v*(4!)", "dQdPlQPl">;
742
743def LD1_LANE : WInst<"vld1_lane", ".(c*!).I", "dQdPlQPl">;
744def LD2_LANE : WInst<"vld2_lane", "2(c*!)2I", "lUlQcQUcQPcQlQUldQdPlQPl">;
745def LD3_LANE : WInst<"vld3_lane", "3(c*!)3I", "lUlQcQUcQPcQlQUldQdPlQPl">;
746def LD4_LANE : WInst<"vld4_lane", "4(c*!)4I", "lUlQcQUcQPcQlQUldQdPlQPl">;
747def ST1_LANE : WInst<"vst1_lane", "v*(.!)I", "dQdPlQPl">;
748def ST2_LANE : WInst<"vst2_lane", "v*(2!)I", "lUlQcQUcQPcQlQUldQdPlQPl">;
749def ST3_LANE : WInst<"vst3_lane", "v*(3!)I", "lUlQcQUcQPcQlQUldQdPlQPl">;
750def ST4_LANE : WInst<"vst4_lane", "v*(4!)I", "lUlQcQUcQPcQlQUldQdPlQPl">;
751
752def LD1_DUP : WInst<"vld1_dup", ".(c*!)", "dQdPlQPl">;
753def LD2_DUP : WInst<"vld2_dup", "2(c*!)", "dQdPlQPl">;
754def LD3_DUP : WInst<"vld3_dup", "3(c*!)", "dQdPlQPl">;
755def LD4_DUP : WInst<"vld4_dup", "4(c*!)", "dQdPlQPl">;
756
757def VLDRQ : WInst<"vldrq", "1(c*!)", "Pk">;
758def VSTRQ : WInst<"vstrq", "v*(1!)", "Pk">;
759
760////////////////////////////////////////////////////////////////////////////////
761// Addition
762def ADD : IOpInst<"vadd", "...", "dQd", OP_ADD>;
763
764////////////////////////////////////////////////////////////////////////////////
765// Subtraction
766def SUB : IOpInst<"vsub", "...", "dQd", OP_SUB>;
767
768////////////////////////////////////////////////////////////////////////////////
769// Multiplication
770def MUL : IOpInst<"vmul", "...", "dQd", OP_MUL>;
771def MLA : IOpInst<"vmla", "....", "dQd", OP_MLA>;
772def MLS : IOpInst<"vmls", "....", "dQd", OP_MLS>;
773
774////////////////////////////////////////////////////////////////////////////////
775// Multiplication Extended
776def MULX : SInst<"vmulx", "...", "fdQfQd">;
777
778////////////////////////////////////////////////////////////////////////////////
779// Division
780def FDIV : IOpInst<"vdiv", "...", "fdQfQd", OP_DIV>;
781
782////////////////////////////////////////////////////////////////////////////////
783// Vector fused multiply-add operations
784def FMLA : SInst<"vfma", "....", "dQd">;
785def FMLS : SOpInst<"vfms", "....", "dQd", OP_FMLS>;
786
787////////////////////////////////////////////////////////////////////////////////
788// MUL, MLA, MLS, FMA, FMS definitions with scalar argument
789def VMUL_N_A64 : IOpInst<"vmul_n", "..1", "Qd", OP_MUL_N>;
790
791def FMLA_N : SOpInst<"vfma_n", "...1", "dQd", OP_FMLA_N>;
792def FMLS_N : SOpInst<"vfms_n", "...1", "fdQfQd", OP_FMLS_N>;
793
794////////////////////////////////////////////////////////////////////////////////
795// Logical operations
796def BSL : SInst<"vbsl", ".U..", "dPlQdQPl">;
797
798////////////////////////////////////////////////////////////////////////////////
799// Absolute Difference
800def ABD : SInst<"vabd", "...", "dQd">;
801
802////////////////////////////////////////////////////////////////////////////////
803// saturating absolute/negate
804def ABS : SInst<"vabs", "..", "dQdlQl">;
805def QABS : SInst<"vqabs", "..", "lQl">;
806def NEG : SOpInst<"vneg", "..", "dlQdQl", OP_NEG>;
807def QNEG : SInst<"vqneg", "..", "lQl">;
808
809////////////////////////////////////////////////////////////////////////////////
810// Signed Saturating Accumulated of Unsigned Value
811def SUQADD : SInst<"vuqadd", "..U", "csilQcQsQiQl">;
812
813////////////////////////////////////////////////////////////////////////////////
814// Unsigned Saturating Accumulated of Signed Value
815def USQADD : SInst<"vsqadd", "..S", "UcUsUiUlQUcQUsQUiQUl">;
816
817////////////////////////////////////////////////////////////////////////////////
818// Reciprocal/Sqrt
819def FRECPS : IInst<"vrecps", "...", "dQd">;
820def FRSQRTS : IInst<"vrsqrts", "...", "dQd">;
821def FRECPE : SInst<"vrecpe", "..", "dQd">;
822def FRSQRTE : SInst<"vrsqrte", "..", "dQd">;
823def FSQRT : SInst<"vsqrt", "..", "fdQfQd">;
824
825////////////////////////////////////////////////////////////////////////////////
826// bitwise reverse
827def RBIT : IInst<"vrbit", "..", "cUcPcQcQUcQPc">;
828
829////////////////////////////////////////////////////////////////////////////////
830// Integer extract and narrow to high
831def XTN2 : SOpInst<"vmovn_high", "(<Q)<Q", "silUsUiUl", OP_XTN>;
832
833////////////////////////////////////////////////////////////////////////////////
834// Signed integer saturating extract and unsigned narrow to high
835def SQXTUN2 : SOpInst<"vqmovun_high", "(<U)(<Uq).", "HsHiHl", OP_SQXTUN>;
836
837////////////////////////////////////////////////////////////////////////////////
838// Integer saturating extract and narrow to high
839def QXTN2 : SOpInst<"vqmovn_high", "(<Q)<Q", "silUsUiUl", OP_QXTN>;
840
841////////////////////////////////////////////////////////////////////////////////
842// Converting vectors
843
844def VCVT_F32_F64 : SInst<"vcvt_f32_f64", "(<q).", "Qd">;
845def VCVT_F64_F32 : SInst<"vcvt_f64_f32", "(>Q).", "f">;
846
847def VCVT_S64 : SInst<"vcvt_s64", "S.", "dQd">;
848def VCVT_U64 : SInst<"vcvt_u64", "U.", "dQd">;
849def VCVT_F64 : SInst<"vcvt_f64", "F(.!)", "lUlQlQUl">;
850
851def VCVT_HIGH_F16_F32 : SOpInst<"vcvt_high_f16", "<(<q!)Q", "Hf", OP_VCVT_NA_HI_F16>;
852def VCVT_HIGH_F32_F16 : SOpInst<"vcvt_high_f32", "(>Q)(Q!)", "h", OP_VCVT_EX_HI_F32>;
853def VCVT_HIGH_F32_F64 : SOpInst<"vcvt_high_f32", "(<Q)(F<!)Q", "d", OP_VCVT_NA_HI_F32>;
854def VCVT_HIGH_F64_F32 : SOpInst<"vcvt_high_f64", "(>Q)(Q!)", "f", OP_VCVT_EX_HI_F64>;
855
856def VCVTX_F32_F64 : SInst<"vcvtx_f32", "(F<)(Q!)", "d">;
857def VCVTX_HIGH_F32_F64 : SOpInst<"vcvtx_high_f32", "(<Q)(F<!)Q", "d", OP_VCVTX_HI>;
858
859////////////////////////////////////////////////////////////////////////////////
860// Comparison
861def FCAGE : IInst<"vcage", "U..", "dQd">;
862def FCAGT : IInst<"vcagt", "U..", "dQd">;
863def FCALE : IInst<"vcale", "U..", "dQd">;
864def FCALT : IInst<"vcalt", "U..", "dQd">;
865def CMTST : WInst<"vtst", "U..", "lUlPlQlQUlQPl">;
866def CFMEQ : SOpInst<"vceq", "U..", "lUldQdQlQUlPlQPl", OP_EQ>;
867def CFMGE : SOpInst<"vcge", "U..", "lUldQdQlQUl", OP_GE>;
868def CFMLE : SOpInst<"vcle", "U..", "lUldQdQlQUl", OP_LE>;
869def CFMGT : SOpInst<"vcgt", "U..", "lUldQdQlQUl", OP_GT>;
870def CFMLT : SOpInst<"vclt", "U..", "lUldQdQlQUl", OP_LT>;
871
872def CMEQ : SInst<"vceqz", "U.",
873 "csilfUcUsUiUlPcPlQcQsQiQlQfQUcQUsQUiQUlQPcdQdQPl">;
874def CMGE : SInst<"vcgez", "U.", "csilfdQcQsQiQlQfQd">;
875def CMLE : SInst<"vclez", "U.", "csilfdQcQsQiQlQfQd">;
876def CMGT : SInst<"vcgtz", "U.", "csilfdQcQsQiQlQfQd">;
877def CMLT : SInst<"vcltz", "U.", "csilfdQcQsQiQlQfQd">;
878
879////////////////////////////////////////////////////////////////////////////////
880// Max/Min Integer
881def MAX : SInst<"vmax", "...", "dQd">;
882def MIN : SInst<"vmin", "...", "dQd">;
883
884////////////////////////////////////////////////////////////////////////////////
885// Pairwise Max/Min
886def MAXP : SInst<"vpmax", "...", "QcQsQiQUcQUsQUiQfQd">;
887def MINP : SInst<"vpmin", "...", "QcQsQiQUcQUsQUiQfQd">;
888
889////////////////////////////////////////////////////////////////////////////////
890// Pairwise MaxNum/MinNum Floating Point
891def FMAXNMP : SInst<"vpmaxnm", "...", "fQfQd">;
892def FMINNMP : SInst<"vpminnm", "...", "fQfQd">;
893
894////////////////////////////////////////////////////////////////////////////////
895// Pairwise Addition
896def ADDP : IInst<"vpadd", "...", "QcQsQiQlQUcQUsQUiQUlQfQd">;
897
898////////////////////////////////////////////////////////////////////////////////
899// Shifts by constant
900let isShift = 1 in {
901// Left shift long high
902def SHLL_HIGH_N : SOpInst<"vshll_high_n", ">.I", "HcHsHiHUcHUsHUi",
903 OP_LONG_HI>;
904
905////////////////////////////////////////////////////////////////////////////////
906def SRI_N : WInst<"vsri_n", "...I", "PlQPl">;
907def SLI_N : WInst<"vsli_n", "...I", "PlQPl">;
908
909// Right shift narrow high
910def SHRN_HIGH_N : IOpInst<"vshrn_high_n", "<(<q).I",
911 "HsHiHlHUsHUiHUl", OP_NARROW_HI>;
912def QSHRUN_HIGH_N : SOpInst<"vqshrun_high_n", "<(<q).I",
913 "HsHiHl", OP_NARROW_HI>;
914def RSHRN_HIGH_N : IOpInst<"vrshrn_high_n", "<(<q).I",
915 "HsHiHlHUsHUiHUl", OP_NARROW_HI>;
916def QRSHRUN_HIGH_N : SOpInst<"vqrshrun_high_n", "<(<q).I",
917 "HsHiHl", OP_NARROW_HI>;
918def QSHRN_HIGH_N : SOpInst<"vqshrn_high_n", "<(<q).I",
919 "HsHiHlHUsHUiHUl", OP_NARROW_HI>;
920def QRSHRN_HIGH_N : SOpInst<"vqrshrn_high_n", "<(<q).I",
921 "HsHiHlHUsHUiHUl", OP_NARROW_HI>;
922}
923
924////////////////////////////////////////////////////////////////////////////////
925// Converting vectors
926def VMOVL_HIGH : SOpInst<"vmovl_high", ">.", "HcHsHiHUcHUsHUi", OP_MOVL_HI>;
927
928let isVCVT_N = 1 in {
929def CVTF_N_F64 : SInst<"vcvt_n_f64", "F(.!)I", "lUlQlQUl">;
930def FCVTZS_N_S64 : SInst<"vcvt_n_s64", "S.I", "dQd">;
931def FCVTZS_N_U64 : SInst<"vcvt_n_u64", "U.I", "dQd">;
932}
933
934////////////////////////////////////////////////////////////////////////////////
935// 3VDiff class using high 64-bit in operands
936def VADDL_HIGH : SOpInst<"vaddl_high", "(>Q)QQ", "csiUcUsUi", OP_ADDLHi>;
937def VADDW_HIGH : SOpInst<"vaddw_high", "(>Q)(>Q)Q", "csiUcUsUi", OP_ADDWHi>;
938def VSUBL_HIGH : SOpInst<"vsubl_high", "(>Q)QQ", "csiUcUsUi", OP_SUBLHi>;
939def VSUBW_HIGH : SOpInst<"vsubw_high", "(>Q)(>Q)Q", "csiUcUsUi", OP_SUBWHi>;
940
941def VABDL_HIGH : SOpInst<"vabdl_high", "(>Q)QQ", "csiUcUsUi", OP_ABDLHi>;
942def VABAL_HIGH : SOpInst<"vabal_high", "(>Q)(>Q)QQ", "csiUcUsUi", OP_ABALHi>;
943
944def VMULL_HIGH : SOpInst<"vmull_high", "(>Q)QQ", "csiUcUsUiPc", OP_MULLHi>;
945def VMULL_HIGH_N : SOpInst<"vmull_high_n", "(>Q)Q1", "siUsUi", OP_MULLHi_N>;
946def VMLAL_HIGH : SOpInst<"vmlal_high", "(>Q)(>Q)QQ", "csiUcUsUi", OP_MLALHi>;
947def VMLAL_HIGH_N : SOpInst<"vmlal_high_n", "(>Q)(>Q)Q1", "siUsUi", OP_MLALHi_N>;
948def VMLSL_HIGH : SOpInst<"vmlsl_high", "(>Q)(>Q)QQ", "csiUcUsUi", OP_MLSLHi>;
949def VMLSL_HIGH_N : SOpInst<"vmlsl_high_n", "(>Q)(>Q)Q1", "siUsUi", OP_MLSLHi_N>;
950
951def VADDHN_HIGH : SOpInst<"vaddhn_high", "(<Q)<QQ", "silUsUiUl", OP_ADDHNHi>;
952def VRADDHN_HIGH : SOpInst<"vraddhn_high", "(<Q)<QQ", "silUsUiUl", OP_RADDHNHi>;
953def VSUBHN_HIGH : SOpInst<"vsubhn_high", "(<Q)<QQ", "silUsUiUl", OP_SUBHNHi>;
954def VRSUBHN_HIGH : SOpInst<"vrsubhn_high", "(<Q)<QQ", "silUsUiUl", OP_RSUBHNHi>;
955
956def VQDMULL_HIGH : SOpInst<"vqdmull_high", "(>Q)QQ", "si", OP_QDMULLHi>;
957def VQDMULL_HIGH_N : SOpInst<"vqdmull_high_n", "(>Q)Q1", "si", OP_QDMULLHi_N>;
958def VQDMLAL_HIGH : SOpInst<"vqdmlal_high", "(>Q)(>Q)QQ", "si", OP_QDMLALHi>;
959def VQDMLAL_HIGH_N : SOpInst<"vqdmlal_high_n", "(>Q)(>Q)Q1", "si", OP_QDMLALHi_N>;
960def VQDMLSL_HIGH : SOpInst<"vqdmlsl_high", "(>Q)(>Q)QQ", "si", OP_QDMLSLHi>;
961def VQDMLSL_HIGH_N : SOpInst<"vqdmlsl_high_n", "(>Q)(>Q)Q1", "si", OP_QDMLSLHi_N>;
962def VMULL_P64 : SInst<"vmull", "(1>)11", "Pl">;
963def VMULL_HIGH_P64 : SOpInst<"vmull_high", "(1>)..", "HPl", OP_MULLHi_P64>;
964
965
966////////////////////////////////////////////////////////////////////////////////
967// Extract or insert element from vector
968def GET_LANE : IInst<"vget_lane", "1.I", "dQdPlQPl">;
969def SET_LANE : IInst<"vset_lane", ".1.I", "dQdPlQPl">;
970def COPY_LANE : IOpInst<"vcopy_lane", "..I.I",
971 "csilUcUsUiUlPcPsPlfd", OP_COPY_LN>;
972def COPYQ_LANE : IOpInst<"vcopy_lane", "..IqI",
973 "QcQsQiQlQUcQUsQUiQUlQPcQPsQfQdQPl", OP_COPY_LN>;
974def COPY_LANEQ : IOpInst<"vcopy_laneq", "..IQI",
975 "csilPcPsPlUcUsUiUlfd", OP_COPY_LN> {
976 let isLaneQ = 1;
977}
978def COPYQ_LANEQ : IOpInst<"vcopy_laneq", "..I.I",
979 "QcQsQiQlQUcQUsQUiQUlQPcQPsQfQdQPl", OP_COPY_LN> {
980 let isLaneQ = 1;
981}
982
983////////////////////////////////////////////////////////////////////////////////
984// Set all lanes to same value
985def VDUP_LANE1: WOpInst<"vdup_lane", ".qI", "hdQhQdPlQPl", OP_DUP_LN>;
986def VDUP_LANE2: WOpInst<"vdup_laneq", ".QI",
987 "csilUcUsUiUlPcPshfdQcQsQiQlQPcQPsQUcQUsQUiQUlQhQfQdPlQPl",
988 OP_DUP_LN> {
989 let isLaneQ = 1;
990}
991def DUP_N : WOpInst<"vdup_n", ".1", "dQdPlQPl", OP_DUP>;
992def MOV_N : WOpInst<"vmov_n", ".1", "dQdPlQPl", OP_DUP>;
993
994////////////////////////////////////////////////////////////////////////////////
995def COMBINE : NoTestOpInst<"vcombine", "Q..", "dPl", OP_CONC>;
996
997////////////////////////////////////////////////////////////////////////////////
998//Initialize a vector from bit pattern
999def CREATE : NoTestOpInst<"vcreate", ".(IU>)", "dPl", OP_CAST> {
1000 let BigEndianSafe = 1;
1001}
1002
1003////////////////////////////////////////////////////////////////////////////////
1004
1005def VMLA_LANEQ : IOpInst<"vmla_laneq", "...QI",
1006 "siUsUifQsQiQUsQUiQf", OP_MLA_LN> {
1007 let isLaneQ = 1;
1008}
1009def VMLS_LANEQ : IOpInst<"vmls_laneq", "...QI",
1010 "siUsUifQsQiQUsQUiQf", OP_MLS_LN> {
1011 let isLaneQ = 1;
1012}
1013
1014def VFMA_LANE : IInst<"vfma_lane", "...qI", "fdQfQd">;
1015def VFMA_LANEQ : IInst<"vfma_laneq", "...QI", "fdQfQd"> {
1016 let isLaneQ = 1;
1017}
1018def VFMS_LANE : IOpInst<"vfms_lane", "...qI", "fdQfQd", OP_FMS_LN>;
1019def VFMS_LANEQ : IOpInst<"vfms_laneq", "...QI", "fdQfQd", OP_FMS_LNQ> {
1020 let isLaneQ = 1;
1021}
1022
1023def VMLAL_LANEQ : SOpInst<"vmlal_laneq", "(>Q)(>Q).QI", "siUsUi", OP_MLAL_LN> {
1024 let isLaneQ = 1;
1025}
1026def VMLAL_HIGH_LANE : SOpInst<"vmlal_high_lane", "(>Q)(>Q)Q.I", "siUsUi",
1027 OP_MLALHi_LN>;
1028def VMLAL_HIGH_LANEQ : SOpInst<"vmlal_high_laneq", "(>Q)(>Q)QQI", "siUsUi",
1029 OP_MLALHi_LN> {
1030 let isLaneQ = 1;
1031}
1032def VMLSL_LANEQ : SOpInst<"vmlsl_laneq", "(>Q)(>Q).QI", "siUsUi", OP_MLSL_LN> {
1033 let isLaneQ = 1;
1034}
1035def VMLSL_HIGH_LANE : SOpInst<"vmlsl_high_lane", "(>Q)(>Q)Q.I", "siUsUi",
1036 OP_MLSLHi_LN>;
1037def VMLSL_HIGH_LANEQ : SOpInst<"vmlsl_high_laneq", "(>Q)(>Q)QQI", "siUsUi",
1038 OP_MLSLHi_LN> {
1039 let isLaneQ = 1;
1040}
1041
1042def VQDMLAL_LANEQ : SOpInst<"vqdmlal_laneq", "(>Q)(>Q).QI", "si", OP_QDMLAL_LN> {
1043 let isLaneQ = 1;
1044}
1045def VQDMLAL_HIGH_LANE : SOpInst<"vqdmlal_high_lane", "(>Q)(>Q)Q.I", "si",
1046 OP_QDMLALHi_LN>;
1047def VQDMLAL_HIGH_LANEQ : SOpInst<"vqdmlal_high_laneq", "(>Q)(>Q)QQI", "si",
1048 OP_QDMLALHi_LN> {
1049 let isLaneQ = 1;
1050}
1051def VQDMLSL_LANEQ : SOpInst<"vqdmlsl_laneq", "(>Q)(>Q).QI", "si", OP_QDMLSL_LN> {
1052 let isLaneQ = 1;
1053}
1054def VQDMLSL_HIGH_LANE : SOpInst<"vqdmlsl_high_lane", "(>Q)(>Q)Q.I", "si",
1055 OP_QDMLSLHi_LN>;
1056def VQDMLSL_HIGH_LANEQ : SOpInst<"vqdmlsl_high_laneq", "(>Q)(>Q)QQI", "si",
1057 OP_QDMLSLHi_LN> {
1058 let isLaneQ = 1;
1059}
1060
1061// Newly add double parameter for vmul_lane in aarch64
1062// Note: d type is handled by SCALAR_VMUL_LANE
1063def VMUL_LANE_A64 : IOpInst<"vmul_lane", "..qI", "Qd", OP_MUL_LN>;
1064
1065// Note: d type is handled by SCALAR_VMUL_LANEQ
1066def VMUL_LANEQ : IOpInst<"vmul_laneq", "..QI",
1067 "sifUsUiQsQiQUsQUiQfQd", OP_MUL_LN> {
1068 let isLaneQ = 1;
1069}
1070def VMULL_LANEQ : SOpInst<"vmull_laneq", "(>Q).QI", "siUsUi", OP_MULL_LN> {
1071 let isLaneQ = 1;
1072}
1073def VMULL_HIGH_LANE : SOpInst<"vmull_high_lane", "(>Q)Q.I", "siUsUi",
1074 OP_MULLHi_LN>;
1075def VMULL_HIGH_LANEQ : SOpInst<"vmull_high_laneq", "(>Q)QQI", "siUsUi",
1076 OP_MULLHi_LN> {
1077 let isLaneQ = 1;
1078}
1079
1080def VQDMULL_LANEQ : SOpInst<"vqdmull_laneq", "(>Q).QI", "si", OP_QDMULL_LN> {
1081 let isLaneQ = 1;
1082}
1083def VQDMULL_HIGH_LANE : SOpInst<"vqdmull_high_lane", "(>Q)Q.I", "si",
1084 OP_QDMULLHi_LN>;
1085def VQDMULL_HIGH_LANEQ : SOpInst<"vqdmull_high_laneq", "(>Q)QQI", "si",
1086 OP_QDMULLHi_LN> {
1087 let isLaneQ = 1;
1088}
1089
1090let isLaneQ = 1 in {
1091def VQDMULH_LANEQ : SInst<"vqdmulh_laneq", "..QI", "siQsQi">;
1092def VQRDMULH_LANEQ : SInst<"vqrdmulh_laneq", "..QI", "siQsQi">;
1093}
1094let ArchGuard = "defined(__ARM_FEATURE_QRDMX) && defined(__aarch64__)" in {
1095def VQRDMLAH_LANEQ : SOpInst<"vqrdmlah_laneq", "...QI", "siQsQi", OP_QRDMLAH_LN> {
1096 let isLaneQ = 1;
1097}
1098def VQRDMLSH_LANEQ : SOpInst<"vqrdmlsh_laneq", "...QI", "siQsQi", OP_QRDMLSH_LN> {
1099 let isLaneQ = 1;
1100}
1101}
1102
1103// Note: d type implemented by SCALAR_VMULX_LANE
1104def VMULX_LANE : IOpInst<"vmulx_lane", "..qI", "fQfQd", OP_MULX_LN>;
1105// Note: d type is implemented by SCALAR_VMULX_LANEQ
1106def VMULX_LANEQ : IOpInst<"vmulx_laneq", "..QI", "fQfQd", OP_MULX_LN> {
1107 let isLaneQ = 1;
1108}
1109
1110////////////////////////////////////////////////////////////////////////////////
1111// Across vectors class
1112def VADDLV : SInst<"vaddlv", "(1>).", "csiUcUsUiQcQsQiQUcQUsQUi">;
1113def VMAXV : SInst<"vmaxv", "1.", "csifUcUsUiQcQsQiQUcQUsQUiQfQd">;
1114def VMINV : SInst<"vminv", "1.", "csifUcUsUiQcQsQiQUcQUsQUiQfQd">;
1115def VADDV : SInst<"vaddv", "1.", "csifUcUsUiQcQsQiQUcQUsQUiQfQdQlQUl">;
1116def FMAXNMV : SInst<"vmaxnmv", "1.", "fQfQd">;
1117def FMINNMV : SInst<"vminnmv", "1.", "fQfQd">;
1118
1119////////////////////////////////////////////////////////////////////////////////
1120// Newly added Vector Extract for f64
1121def VEXT_A64 : WInst<"vext", "...I", "dQdPlQPl">;
1122
1123////////////////////////////////////////////////////////////////////////////////
1124// Crypto
1125let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_AES)" in {
1126def AESE : SInst<"vaese", "...", "QUc">;
1127def AESD : SInst<"vaesd", "...", "QUc">;
1128def AESMC : SInst<"vaesmc", "..", "QUc">;
1129def AESIMC : SInst<"vaesimc", "..", "QUc">;
1130}
1131
1132let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_SHA2)" in {
1133def SHA1H : SInst<"vsha1h", "11", "Ui">;
1134def SHA1SU1 : SInst<"vsha1su1", "...", "QUi">;
1135def SHA256SU0 : SInst<"vsha256su0", "...", "QUi">;
1136
1137def SHA1C : SInst<"vsha1c", "..1.", "QUi">;
1138def SHA1P : SInst<"vsha1p", "..1.", "QUi">;
1139def SHA1M : SInst<"vsha1m", "..1.", "QUi">;
1140def SHA1SU0 : SInst<"vsha1su0", "....", "QUi">;
1141def SHA256H : SInst<"vsha256h", "....", "QUi">;
1142def SHA256H2 : SInst<"vsha256h2", "....", "QUi">;
1143def SHA256SU1 : SInst<"vsha256su1", "....", "QUi">;
1144}
1145
1146let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_SHA3) && defined(__aarch64__)" in {
1147def BCAX : SInst<"vbcax", "....", "QUcQUsQUiQUlQcQsQiQl">;
1148def EOR3 : SInst<"veor3", "....", "QUcQUsQUiQUlQcQsQiQl">;
1149def RAX1 : SInst<"vrax1", "...", "QUl">;
1150
1151let isVXAR = 1 in {
1152def XAR : SInst<"vxar", "...I", "QUl">;
1153}
1154}
1155
1156let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_SHA512) && defined(__aarch64__)" in {
1157
1158def SHA512SU0 : SInst<"vsha512su0", "...", "QUl">;
1159def SHA512su1 : SInst<"vsha512su1", "....", "QUl">;
1160def SHA512H : SInst<"vsha512h", "....", "QUl">;
1161def SHA512H2 : SInst<"vsha512h2", "....", "QUl">;
1162}
1163
1164let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_SM3) && defined(__aarch64__)" in {
1165def SM3SS1 : SInst<"vsm3ss1", "....", "QUi">;
1166def SM3TT1A : SInst<"vsm3tt1a", "....I", "QUi">;
1167def SM3TT1B : SInst<"vsm3tt1b", "....I", "QUi">;
1168def SM3TT2A : SInst<"vsm3tt2a", "....I", "QUi">;
1169def SM3TT2B : SInst<"vsm3tt2b", "....I", "QUi">;
1170def SM3PARTW1 : SInst<"vsm3partw1", "....", "QUi">;
1171def SM3PARTW2 : SInst<"vsm3partw2", "....", "QUi">;
1172}
1173
1174let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_SM4) && defined(__aarch64__)" in {
1175def SM4E : SInst<"vsm4e", "...", "QUi">;
1176def SM4EKEY : SInst<"vsm4ekey", "...", "QUi">;
1177}
1178
1179////////////////////////////////////////////////////////////////////////////////
1180// poly128_t vadd for AArch64 only see VADDP for the rest
1181def VADDP_Q : WInst<"vadd", "...", "QPk">;
1182
1183////////////////////////////////////////////////////////////////////////////////
1184// Float -> Int conversions with explicit rounding mode
1185
1186let ArchGuard = "__ARM_ARCH >= 8" in {
1187def FCVTNS_S32 : SInst<"vcvtn_s32", "S.", "fQf">;
1188def FCVTNU_S32 : SInst<"vcvtn_u32", "U.", "fQf">;
1189def FCVTPS_S32 : SInst<"vcvtp_s32", "S.", "fQf">;
1190def FCVTPU_S32 : SInst<"vcvtp_u32", "U.", "fQf">;
1191def FCVTMS_S32 : SInst<"vcvtm_s32", "S.", "fQf">;
1192def FCVTMU_S32 : SInst<"vcvtm_u32", "U.", "fQf">;
1193def FCVTAS_S32 : SInst<"vcvta_s32", "S.", "fQf">;
1194def FCVTAU_S32 : SInst<"vcvta_u32", "U.", "fQf">;
1195}
1196
1197let ArchGuard = "__ARM_ARCH >= 8 && defined(__aarch64__)" in {
1198def FCVTNS_S64 : SInst<"vcvtn_s64", "S.", "dQd">;
1199def FCVTNU_S64 : SInst<"vcvtn_u64", "U.", "dQd">;
1200def FCVTPS_S64 : SInst<"vcvtp_s64", "S.", "dQd">;
1201def FCVTPU_S64 : SInst<"vcvtp_u64", "U.", "dQd">;
1202def FCVTMS_S64 : SInst<"vcvtm_s64", "S.", "dQd">;
1203def FCVTMU_S64 : SInst<"vcvtm_u64", "U.", "dQd">;
1204def FCVTAS_S64 : SInst<"vcvta_s64", "S.", "dQd">;
1205def FCVTAU_S64 : SInst<"vcvta_u64", "U.", "dQd">;
1206}
1207
1208////////////////////////////////////////////////////////////////////////////////
1209// Round to Integral
1210
1211let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_DIRECTED_ROUNDING)" in {
1212def FRINTN_S32 : SInst<"vrndn", "..", "fQf">;
1213def FRINTA_S32 : SInst<"vrnda", "..", "fQf">;
1214def FRINTP_S32 : SInst<"vrndp", "..", "fQf">;
1215def FRINTM_S32 : SInst<"vrndm", "..", "fQf">;
1216def FRINTX_S32 : SInst<"vrndx", "..", "fQf">;
1217def FRINTZ_S32 : SInst<"vrnd", "..", "fQf">;
1218def FRINTI_S32 : SInst<"vrndi", "..", "fQf">;
1219}
1220
1221let ArchGuard = "__ARM_ARCH >= 8 && defined(__aarch64__) && defined(__ARM_FEATURE_DIRECTED_ROUNDING)" in {
1222def FRINTN_S64 : SInst<"vrndn", "..", "dQd">;
1223def FRINTA_S64 : SInst<"vrnda", "..", "dQd">;
1224def FRINTP_S64 : SInst<"vrndp", "..", "dQd">;
1225def FRINTM_S64 : SInst<"vrndm", "..", "dQd">;
1226def FRINTX_S64 : SInst<"vrndx", "..", "dQd">;
1227def FRINTZ_S64 : SInst<"vrnd", "..", "dQd">;
1228def FRINTI_S64 : SInst<"vrndi", "..", "dQd">;
1229}
1230
1231let ArchGuard = "__ARM_ARCH >= 8 && defined(__aarch64__) && defined(__ARM_FEATURE_FRINT)" in {
1232def FRINT32X_S32 : SInst<"vrnd32x", "..", "fQf">;
1233def FRINT32Z_S32 : SInst<"vrnd32z", "..", "fQf">;
1234def FRINT64X_S32 : SInst<"vrnd64x", "..", "fQf">;
1235def FRINT64Z_S32 : SInst<"vrnd64z", "..", "fQf">;
1236}
1237
1238////////////////////////////////////////////////////////////////////////////////
1239// MaxNum/MinNum Floating Point
1240
1241let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_NUMERIC_MAXMIN)" in {
1242def FMAXNM_S32 : SInst<"vmaxnm", "...", "fQf">;
1243def FMINNM_S32 : SInst<"vminnm", "...", "fQf">;
1244}
1245
1246let ArchGuard = "__ARM_ARCH >= 8 && defined(__aarch64__) && defined(__ARM_FEATURE_NUMERIC_MAXMIN)" in {
1247def FMAXNM_S64 : SInst<"vmaxnm", "...", "dQd">;
1248def FMINNM_S64 : SInst<"vminnm", "...", "dQd">;
1249}
1250
1251////////////////////////////////////////////////////////////////////////////////
1252// Permutation
1253def VTRN1 : SOpInst<"vtrn1", "...",
1254 "csiUcUsUifPcPsQcQsQiQlQUcQUsQUiQUlQfQdQPcQPsQPl", OP_TRN1>;
1255def VZIP1 : SOpInst<"vzip1", "...",
1256 "csiUcUsUifPcPsQcQsQiQlQUcQUsQUiQUlQfQdQPcQPsQPl", OP_ZIP1>;
1257def VUZP1 : SOpInst<"vuzp1", "...",
1258 "csiUcUsUifPcPsQcQsQiQlQUcQUsQUiQUlQfQdQPcQPsQPl", OP_UZP1>;
1259def VTRN2 : SOpInst<"vtrn2", "...",
1260 "csiUcUsUifPcPsQcQsQiQlQUcQUsQUiQUlQfQdQPcQPsQPl", OP_TRN2>;
1261def VZIP2 : SOpInst<"vzip2", "...",
1262 "csiUcUsUifPcPsQcQsQiQlQUcQUsQUiQUlQfQdQPcQPsQPl", OP_ZIP2>;
1263def VUZP2 : SOpInst<"vuzp2", "...",
1264 "csiUcUsUifPcPsQcQsQiQlQUcQUsQUiQUlQfQdQPcQPsQPl", OP_UZP2>;
1265
1266////////////////////////////////////////////////////////////////////////////////
1267// Table lookup
1268let InstName = "vtbl" in {
1269def VQTBL1_A64 : WInst<"vqtbl1", ".QU", "UccPcQUcQcQPc">;
1270def VQTBL2_A64 : WInst<"vqtbl2", ".(2Q)U", "UccPcQUcQcQPc">;
1271def VQTBL3_A64 : WInst<"vqtbl3", ".(3Q)U", "UccPcQUcQcQPc">;
1272def VQTBL4_A64 : WInst<"vqtbl4", ".(4Q)U", "UccPcQUcQcQPc">;
1273}
1274let InstName = "vtbx" in {
1275def VQTBX1_A64 : WInst<"vqtbx1", "..QU", "UccPcQUcQcQPc">;
1276def VQTBX2_A64 : WInst<"vqtbx2", "..(2Q)U", "UccPcQUcQcQPc">;
1277def VQTBX3_A64 : WInst<"vqtbx3", "..(3Q)U", "UccPcQUcQcQPc">;
1278def VQTBX4_A64 : WInst<"vqtbx4", "..(4Q)U", "UccPcQUcQcQPc">;
1279}
1280
1281////////////////////////////////////////////////////////////////////////////////
1282// Vector reinterpret cast operations
1283
1284// NeonEmitter implicitly takes the cartesian product of the type string with
1285// itself during generation so, unlike all other intrinsics, this one should
1286// include *all* types, not just additional ones.
1287def VVREINTERPRET : REINTERPRET_CROSS_SELF<"csilUcUsUiUlhfdPcPsPlQcQsQiQlQUcQUsQUiQUlQhQfQdQPcQPsQPlQPk"> {
1288 let ArchGuard = "__ARM_ARCH >= 8 && defined(__aarch64__)";
1289 let BigEndianSafe = 1;
1290}
1291
1292////////////////////////////////////////////////////////////////////////////////
1293// Scalar Intrinsics
1294// Scalar Arithmetic
1295
1296// Scalar Addition
1297def SCALAR_ADD : SInst<"vadd", "111", "SlSUl">;
1298// Scalar Saturating Add
1299def SCALAR_QADD : SInst<"vqadd", "111", "ScSsSiSlSUcSUsSUiSUl">;
1300
1301// Scalar Subtraction
1302def SCALAR_SUB : SInst<"vsub", "111", "SlSUl">;
1303// Scalar Saturating Sub
1304def SCALAR_QSUB : SInst<"vqsub", "111", "ScSsSiSlSUcSUsSUiSUl">;
1305
1306let InstName = "vmov" in {
1307def VGET_HIGH_A64 : NoTestOpInst<"vget_high", ".Q", "dPl", OP_HI>;
1308def VGET_LOW_A64 : NoTestOpInst<"vget_low", ".Q", "dPl", OP_LO>;
1309}
1310
1311////////////////////////////////////////////////////////////////////////////////
1312// Scalar Shift
1313// Scalar Shift Left
1314def SCALAR_SHL: SInst<"vshl", "11(S1)", "SlSUl">;
1315// Scalar Saturating Shift Left
1316def SCALAR_QSHL: SInst<"vqshl", "11(S1)", "ScSsSiSlSUcSUsSUiSUl">;
1317// Scalar Saturating Rounding Shift Left
1318def SCALAR_QRSHL: SInst<"vqrshl", "11(S1)", "ScSsSiSlSUcSUsSUiSUl">;
1319// Scalar Shift Rounding Left
1320def SCALAR_RSHL: SInst<"vrshl", "11(S1)", "SlSUl">;
1321
1322////////////////////////////////////////////////////////////////////////////////
1323// Scalar Shift (Immediate)
1324let isScalarShift = 1 in {
1325// Signed/Unsigned Shift Right (Immediate)
1326def SCALAR_SSHR_N: SInst<"vshr_n", "11I", "SlSUl">;
1327// Signed/Unsigned Rounding Shift Right (Immediate)
1328def SCALAR_SRSHR_N: SInst<"vrshr_n", "11I", "SlSUl">;
1329
1330// Signed/Unsigned Shift Right and Accumulate (Immediate)
1331def SCALAR_SSRA_N: SInst<"vsra_n", "111I", "SlSUl">;
1332// Signed/Unsigned Rounding Shift Right and Accumulate (Immediate)
1333def SCALAR_SRSRA_N: SInst<"vrsra_n", "111I", "SlSUl">;
1334
1335// Shift Left (Immediate)
1336def SCALAR_SHL_N: SInst<"vshl_n", "11I", "SlSUl">;
1337// Signed/Unsigned Saturating Shift Left (Immediate)
1338def SCALAR_SQSHL_N: SInst<"vqshl_n", "11I", "ScSsSiSlSUcSUsSUiSUl">;
1339// Signed Saturating Shift Left Unsigned (Immediate)
1340def SCALAR_SQSHLU_N: SInst<"vqshlu_n", "11I", "ScSsSiSl">;
1341
1342// Shift Right And Insert (Immediate)
1343def SCALAR_SRI_N: SInst<"vsri_n", "111I", "SlSUl">;
1344// Shift Left And Insert (Immediate)
1345def SCALAR_SLI_N: SInst<"vsli_n", "111I", "SlSUl">;
1346
1347let isScalarNarrowShift = 1 in {
1348 // Signed/Unsigned Saturating Shift Right Narrow (Immediate)
1349 def SCALAR_SQSHRN_N: SInst<"vqshrn_n", "(1<)1I", "SsSiSlSUsSUiSUl">;
1350 // Signed/Unsigned Saturating Rounded Shift Right Narrow (Immediate)
1351 def SCALAR_SQRSHRN_N: SInst<"vqrshrn_n", "(1<)1I", "SsSiSlSUsSUiSUl">;
1352 // Signed Saturating Shift Right Unsigned Narrow (Immediate)
1353 def SCALAR_SQSHRUN_N: SInst<"vqshrun_n", "(1<)1I", "SsSiSl">;
1354 // Signed Saturating Rounded Shift Right Unsigned Narrow (Immediate)
1355 def SCALAR_SQRSHRUN_N: SInst<"vqrshrun_n", "(1<)1I", "SsSiSl">;
1356}
1357
1358////////////////////////////////////////////////////////////////////////////////
1359// Scalar Signed/Unsigned Fixed-point Convert To Floating-Point (Immediate)
1360def SCALAR_SCVTF_N_F32: SInst<"vcvt_n_f32", "(1F)(1!)I", "SiSUi">;
1361def SCALAR_SCVTF_N_F64: SInst<"vcvt_n_f64", "(1F)(1!)I", "SlSUl">;
1362
1363////////////////////////////////////////////////////////////////////////////////
1364// Scalar Floating-point Convert To Signed/Unsigned Fixed-point (Immediate)
1365def SCALAR_FCVTZS_N_S32 : SInst<"vcvt_n_s32", "(1S)1I", "Sf">;
1366def SCALAR_FCVTZU_N_U32 : SInst<"vcvt_n_u32", "(1U)1I", "Sf">;
1367def SCALAR_FCVTZS_N_S64 : SInst<"vcvt_n_s64", "(1S)1I", "Sd">;
1368def SCALAR_FCVTZU_N_U64 : SInst<"vcvt_n_u64", "(1U)1I", "Sd">;
1369}
1370
1371////////////////////////////////////////////////////////////////////////////////
1372// Scalar Floating-point Round to Integral
1373let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_DIRECTED_ROUNDING)" in {
1374def SCALAR_FRINTN_S32 : SInst<"vrndn", "11", "Sf">;
1375}
1376
1377////////////////////////////////////////////////////////////////////////////////
1378// Scalar Reduce Pairwise Addition (Scalar and Floating Point)
1379def SCALAR_ADDP : SInst<"vpadd", "1.", "SfSHlSHdSHUl">;
1380
1381////////////////////////////////////////////////////////////////////////////////
1382// Scalar Reduce Floating Point Pairwise Max/Min
1383def SCALAR_FMAXP : SInst<"vpmax", "1.", "SfSQd">;
1384
1385def SCALAR_FMINP : SInst<"vpmin", "1.", "SfSQd">;
1386
1387////////////////////////////////////////////////////////////////////////////////
1388// Scalar Reduce Floating Point Pairwise maxNum/minNum
1389def SCALAR_FMAXNMP : SInst<"vpmaxnm", "1.", "SfSQd">;
1390def SCALAR_FMINNMP : SInst<"vpminnm", "1.", "SfSQd">;
1391
1392////////////////////////////////////////////////////////////////////////////////
1393// Scalar Integer Saturating Doubling Multiply Half High
1394def SCALAR_SQDMULH : SInst<"vqdmulh", "111", "SsSi">;
1395
1396////////////////////////////////////////////////////////////////////////////////
1397// Scalar Integer Saturating Rounding Doubling Multiply Half High
1398def SCALAR_SQRDMULH : SInst<"vqrdmulh", "111", "SsSi">;
1399
1400let ArchGuard = "defined(__ARM_FEATURE_QRDMX) && defined(__aarch64__)" in {
1401////////////////////////////////////////////////////////////////////////////////
1402// Signed Saturating Rounding Doubling Multiply Accumulate Returning High Half
1403def SCALAR_SQRDMLAH : SOpInst<"vqrdmlah", "1111", "SsSi", OP_QRDMLAH>;
1404
1405////////////////////////////////////////////////////////////////////////////////
1406// Signed Saturating Rounding Doubling Multiply Subtract Returning High Half
1407def SCALAR_SQRDMLSH : SOpInst<"vqrdmlsh", "1111", "SsSi", OP_QRDMLSH>;
1408}
1409
1410////////////////////////////////////////////////////////////////////////////////
1411// Scalar Floating-point Multiply Extended
1412def SCALAR_FMULX : IInst<"vmulx", "111", "SfSd">;
1413
1414////////////////////////////////////////////////////////////////////////////////
1415// Scalar Floating-point Reciprocal Step
1416def SCALAR_FRECPS : IInst<"vrecps", "111", "SfSd">;
1417
1418////////////////////////////////////////////////////////////////////////////////
1419// Scalar Floating-point Reciprocal Square Root Step
1420def SCALAR_FRSQRTS : IInst<"vrsqrts", "111", "SfSd">;
1421
1422////////////////////////////////////////////////////////////////////////////////
1423// Scalar Signed Integer Convert To Floating-point
1424def SCALAR_SCVTFS : SInst<"vcvt_f32", "(1F)(1!)", "Si">;
1425def SCALAR_SCVTFD : SInst<"vcvt_f64", "(1F)(1!)", "Sl">;
1426
1427////////////////////////////////////////////////////////////////////////////////
1428// Scalar Unsigned Integer Convert To Floating-point
1429def SCALAR_UCVTFS : SInst<"vcvt_f32", "(1F)(1!)", "SUi">;
1430def SCALAR_UCVTFD : SInst<"vcvt_f64", "(1F)(1!)", "SUl">;
1431
1432////////////////////////////////////////////////////////////////////////////////
1433// Scalar Floating-point Converts
1434def SCALAR_FCVTXN : IInst<"vcvtx_f32", "(1F<)(1!)", "Sd">;
1435def SCALAR_FCVTNSS : SInst<"vcvtn_s32", "(1S)1", "Sf">;
1436def SCALAR_FCVTNUS : SInst<"vcvtn_u32", "(1U)1", "Sf">;
1437def SCALAR_FCVTNSD : SInst<"vcvtn_s64", "(1S)1", "Sd">;
1438def SCALAR_FCVTNUD : SInst<"vcvtn_u64", "(1U)1", "Sd">;
1439def SCALAR_FCVTMSS : SInst<"vcvtm_s32", "(1S)1", "Sf">;
1440def SCALAR_FCVTMUS : SInst<"vcvtm_u32", "(1U)1", "Sf">;
1441def SCALAR_FCVTMSD : SInst<"vcvtm_s64", "(1S)1", "Sd">;
1442def SCALAR_FCVTMUD : SInst<"vcvtm_u64", "(1U)1", "Sd">;
1443def SCALAR_FCVTASS : SInst<"vcvta_s32", "(1S)1", "Sf">;
1444def SCALAR_FCVTAUS : SInst<"vcvta_u32", "(1U)1", "Sf">;
1445def SCALAR_FCVTASD : SInst<"vcvta_s64", "(1S)1", "Sd">;
1446def SCALAR_FCVTAUD : SInst<"vcvta_u64", "(1U)1", "Sd">;
1447def SCALAR_FCVTPSS : SInst<"vcvtp_s32", "(1S)1", "Sf">;
1448def SCALAR_FCVTPUS : SInst<"vcvtp_u32", "(1U)1", "Sf">;
1449def SCALAR_FCVTPSD : SInst<"vcvtp_s64", "(1S)1", "Sd">;
1450def SCALAR_FCVTPUD : SInst<"vcvtp_u64", "(1U)1", "Sd">;
1451def SCALAR_FCVTZSS : SInst<"vcvt_s32", "(1S)1", "Sf">;
1452def SCALAR_FCVTZUS : SInst<"vcvt_u32", "(1U)1", "Sf">;
1453def SCALAR_FCVTZSD : SInst<"vcvt_s64", "(1S)1", "Sd">;
1454def SCALAR_FCVTZUD : SInst<"vcvt_u64", "(1U)1", "Sd">;
1455
1456////////////////////////////////////////////////////////////////////////////////
1457// Scalar Floating-point Reciprocal Estimate
1458def SCALAR_FRECPE : IInst<"vrecpe", "11", "SfSd">;
1459
1460////////////////////////////////////////////////////////////////////////////////
1461// Scalar Floating-point Reciprocal Exponent
1462def SCALAR_FRECPX : IInst<"vrecpx", "11", "SfSd">;
1463
1464////////////////////////////////////////////////////////////////////////////////
1465// Scalar Floating-point Reciprocal Square Root Estimate
1466def SCALAR_FRSQRTE : IInst<"vrsqrte", "11", "SfSd">;
1467
1468////////////////////////////////////////////////////////////////////////////////
1469// Scalar Integer Comparison
1470def SCALAR_CMEQ : SInst<"vceq", "(U1)11", "SlSUl">;
1471def SCALAR_CMEQZ : SInst<"vceqz", "(U1)1", "SlSUl">;
1472def SCALAR_CMGE : SInst<"vcge", "(U1)11", "Sl">;
1473def SCALAR_CMGEZ : SInst<"vcgez", "(U1)1", "Sl">;
1474def SCALAR_CMHS : SInst<"vcge", "(U1)11", "SUl">;
1475def SCALAR_CMLE : SInst<"vcle", "(U1)11", "SlSUl">;
1476def SCALAR_CMLEZ : SInst<"vclez", "(U1)1", "Sl">;
1477def SCALAR_CMLT : SInst<"vclt", "(U1)11", "SlSUl">;
1478def SCALAR_CMLTZ : SInst<"vcltz", "(U1)1", "Sl">;
1479def SCALAR_CMGT : SInst<"vcgt", "(U1)11", "Sl">;
1480def SCALAR_CMGTZ : SInst<"vcgtz", "(U1)1", "Sl">;
1481def SCALAR_CMHI : SInst<"vcgt", "(U1)11", "SUl">;
1482def SCALAR_CMTST : SInst<"vtst", "(U1)11", "SlSUl">;
1483
1484////////////////////////////////////////////////////////////////////////////////
1485// Scalar Floating-point Comparison
1486def SCALAR_FCMEQ : IInst<"vceq", "(1U)11", "SfSd">;
1487def SCALAR_FCMEQZ : IInst<"vceqz", "(1U)1", "SfSd">;
1488def SCALAR_FCMGE : IInst<"vcge", "(1U)11", "SfSd">;
1489def SCALAR_FCMGEZ : IInst<"vcgez", "(1U)1", "SfSd">;
1490def SCALAR_FCMGT : IInst<"vcgt", "(1U)11", "SfSd">;
1491def SCALAR_FCMGTZ : IInst<"vcgtz", "(1U)1", "SfSd">;
1492def SCALAR_FCMLE : IInst<"vcle", "(1U)11", "SfSd">;
1493def SCALAR_FCMLEZ : IInst<"vclez", "(1U)1", "SfSd">;
1494def SCALAR_FCMLT : IInst<"vclt", "(1U)11", "SfSd">;
1495def SCALAR_FCMLTZ : IInst<"vcltz", "(1U)1", "SfSd">;
1496
1497////////////////////////////////////////////////////////////////////////////////
1498// Scalar Floating-point Absolute Compare Mask Greater Than Or Equal
1499def SCALAR_FACGE : IInst<"vcage", "(1U)11", "SfSd">;
1500def SCALAR_FACLE : IInst<"vcale", "(1U)11", "SfSd">;
1501
1502////////////////////////////////////////////////////////////////////////////////
1503// Scalar Floating-point Absolute Compare Mask Greater Than
1504def SCALAR_FACGT : IInst<"vcagt", "(1U)11", "SfSd">;
1505def SCALAR_FACLT : IInst<"vcalt", "(1U)11", "SfSd">;
1506
1507////////////////////////////////////////////////////////////////////////////////
1508// Scalar Absolute Value
1509def SCALAR_ABS : SInst<"vabs", "11", "Sl">;
1510
1511////////////////////////////////////////////////////////////////////////////////
1512// Scalar Absolute Difference
1513def SCALAR_ABD : IInst<"vabd", "111", "SfSd">;
1514
1515////////////////////////////////////////////////////////////////////////////////
1516// Scalar Signed Saturating Absolute Value
1517def SCALAR_SQABS : SInst<"vqabs", "11", "ScSsSiSl">;
1518
1519////////////////////////////////////////////////////////////////////////////////
1520// Scalar Negate
1521def SCALAR_NEG : SInst<"vneg", "11", "Sl">;
1522
1523////////////////////////////////////////////////////////////////////////////////
1524// Scalar Signed Saturating Negate
1525def SCALAR_SQNEG : SInst<"vqneg", "11", "ScSsSiSl">;
1526
1527////////////////////////////////////////////////////////////////////////////////
1528// Scalar Signed Saturating Accumulated of Unsigned Value
1529def SCALAR_SUQADD : SInst<"vuqadd", "11(1U)", "ScSsSiSl">;
1530
1531////////////////////////////////////////////////////////////////////////////////
1532// Scalar Unsigned Saturating Accumulated of Signed Value
1533def SCALAR_USQADD : SInst<"vsqadd", "11(1S)", "SUcSUsSUiSUl">;
1534
1535////////////////////////////////////////////////////////////////////////////////
1536// Signed Saturating Doubling Multiply-Add Long
1537def SCALAR_SQDMLAL : SInst<"vqdmlal", "(1>)(1>)11", "SsSi">;
1538
1539////////////////////////////////////////////////////////////////////////////////
1540// Signed Saturating Doubling Multiply-Subtract Long
1541def SCALAR_SQDMLSL : SInst<"vqdmlsl", "(1>)(1>)11", "SsSi">;
1542
1543////////////////////////////////////////////////////////////////////////////////
1544// Signed Saturating Doubling Multiply Long
1545def SCALAR_SQDMULL : SInst<"vqdmull", "(1>)11", "SsSi">;
1546
1547////////////////////////////////////////////////////////////////////////////////
1548// Scalar Signed Saturating Extract Unsigned Narrow
1549def SCALAR_SQXTUN : SInst<"vqmovun", "(U1<)1", "SsSiSl">;
1550
1551////////////////////////////////////////////////////////////////////////////////
1552// Scalar Signed Saturating Extract Narrow
1553def SCALAR_SQXTN : SInst<"vqmovn", "(1<)1", "SsSiSl">;
1554
1555////////////////////////////////////////////////////////////////////////////////
1556// Scalar Unsigned Saturating Extract Narrow
1557def SCALAR_UQXTN : SInst<"vqmovn", "(1<)1", "SUsSUiSUl">;
1558
1559// Scalar Floating Point multiply (scalar, by element)
1560def SCALAR_FMUL_LANE : IOpInst<"vmul_lane", "11.I", "SfSd", OP_SCALAR_MUL_LN>;
1561def SCALAR_FMUL_LANEQ : IOpInst<"vmul_laneq", "11QI", "SfSd", OP_SCALAR_MUL_LN> {
1562 let isLaneQ = 1;
1563}
1564
1565// Scalar Floating Point multiply extended (scalar, by element)
1566def SCALAR_FMULX_LANE : IOpInst<"vmulx_lane", "11.I", "SfSd", OP_SCALAR_MULX_LN>;
1567def SCALAR_FMULX_LANEQ : IOpInst<"vmulx_laneq", "11QI", "SfSd", OP_SCALAR_MULX_LN> {
1568 let isLaneQ = 1;
1569}
1570
1571def SCALAR_VMUL_N : IInst<"vmul_n", "..1", "d">;
1572
1573// VMUL_LANE_A64 d type implemented using scalar mul lane
1574def SCALAR_VMUL_LANE : IInst<"vmul_lane", "..qI", "d">;
1575
1576// VMUL_LANEQ d type implemented using scalar mul lane
1577def SCALAR_VMUL_LANEQ : IInst<"vmul_laneq", "..QI", "d"> {
1578 let isLaneQ = 1;
1579}
1580
1581// VMULX_LANE d type implemented using scalar vmulx_lane
1582def SCALAR_VMULX_LANE : IOpInst<"vmulx_lane", "..qI", "d", OP_SCALAR_VMULX_LN>;
1583
1584// VMULX_LANEQ d type implemented using scalar vmulx_laneq
1585def SCALAR_VMULX_LANEQ : IOpInst<"vmulx_laneq", "..QI", "d", OP_SCALAR_VMULX_LNQ> {
1586 let isLaneQ = 1;
1587}
1588
1589// Scalar Floating Point fused multiply-add (scalar, by element)
1590def SCALAR_FMLA_LANE : IInst<"vfma_lane", "111.I", "SfSd">;
1591def SCALAR_FMLA_LANEQ : IInst<"vfma_laneq", "111QI", "SfSd"> {
1592 let isLaneQ = 1;
1593}
1594
1595// Scalar Floating Point fused multiply-subtract (scalar, by element)
1596def SCALAR_FMLS_LANE : IOpInst<"vfms_lane", "111.I", "SfSd", OP_FMS_LN>;
1597def SCALAR_FMLS_LANEQ : IOpInst<"vfms_laneq", "111QI", "SfSd", OP_FMS_LNQ> {
1598 let isLaneQ = 1;
1599}
1600
1601// Signed Saturating Doubling Multiply Long (scalar by element)
1602def SCALAR_SQDMULL_LANE : SOpInst<"vqdmull_lane", "(1>)1.I", "SsSi", OP_SCALAR_QDMULL_LN>;
1603def SCALAR_SQDMULL_LANEQ : SOpInst<"vqdmull_laneq", "(1>)1QI", "SsSi", OP_SCALAR_QDMULL_LN> {
1604 let isLaneQ = 1;
1605}
1606
1607// Signed Saturating Doubling Multiply-Add Long (scalar by element)
1608def SCALAR_SQDMLAL_LANE : SInst<"vqdmlal_lane", "(1>)(1>)1.I", "SsSi">;
1609def SCALAR_SQDMLAL_LANEQ : SInst<"vqdmlal_laneq", "(1>)(1>)1QI", "SsSi"> {
1610 let isLaneQ = 1;
1611}
1612
1613// Signed Saturating Doubling Multiply-Subtract Long (scalar by element)
1614def SCALAR_SQDMLS_LANE : SInst<"vqdmlsl_lane", "(1>)(1>)1.I", "SsSi">;
1615def SCALAR_SQDMLS_LANEQ : SInst<"vqdmlsl_laneq", "(1>)(1>)1QI", "SsSi"> {
1616 let isLaneQ = 1;
1617}
1618
1619// Scalar Integer Saturating Doubling Multiply Half High (scalar by element)
1620def SCALAR_SQDMULH_LANE : SOpInst<"vqdmulh_lane", "11.I", "SsSi", OP_SCALAR_QDMULH_LN>;
1621def SCALAR_SQDMULH_LANEQ : SOpInst<"vqdmulh_laneq", "11QI", "SsSi", OP_SCALAR_QDMULH_LN> {
1622 let isLaneQ = 1;
1623}
1624
1625// Scalar Integer Saturating Rounding Doubling Multiply Half High
1626def SCALAR_SQRDMULH_LANE : SOpInst<"vqrdmulh_lane", "11.I", "SsSi", OP_SCALAR_QRDMULH_LN>;
1627def SCALAR_SQRDMULH_LANEQ : SOpInst<"vqrdmulh_laneq", "11QI", "SsSi", OP_SCALAR_QRDMULH_LN> {
1628 let isLaneQ = 1;
1629}
1630
1631let ArchGuard = "defined(__ARM_FEATURE_QRDMX) && defined(__aarch64__)" in {
1632// Signed Saturating Rounding Doubling Multiply Accumulate Returning High Half
1633def SCALAR_SQRDMLAH_LANE : SOpInst<"vqrdmlah_lane", "111.I", "SsSi", OP_SCALAR_QRDMLAH_LN>;
1634def SCALAR_SQRDMLAH_LANEQ : SOpInst<"vqrdmlah_laneq", "111QI", "SsSi", OP_SCALAR_QRDMLAH_LN> {
1635 let isLaneQ = 1;
1636}
1637
1638// Signed Saturating Rounding Doubling Multiply Subtract Returning High Half
1639def SCALAR_SQRDMLSH_LANE : SOpInst<"vqrdmlsh_lane", "111.I", "SsSi", OP_SCALAR_QRDMLSH_LN>;
1640def SCALAR_SQRDMLSH_LANEQ : SOpInst<"vqrdmlsh_laneq", "111QI", "SsSi", OP_SCALAR_QRDMLSH_LN> {
1641 let isLaneQ = 1;
1642}
1643}
1644
1645def SCALAR_VDUP_LANE : IInst<"vdup_lane", "1.I", "ScSsSiSlSfSdSUcSUsSUiSUlSPcSPs">;
1646def SCALAR_VDUP_LANEQ : IInst<"vdup_laneq", "1QI", "ScSsSiSlSfSdSUcSUsSUiSUlSPcSPs"> {
1647 let isLaneQ = 1;
1648}
1649}
1650
1651// ARMv8.2-A FP16 vector intrinsics for A32/A64.
1652let ArchGuard = "defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)" in {
1653
1654 // ARMv8.2-A FP16 one-operand vector intrinsics.
1655
1656 // Comparison
1657 def CMEQH : SInst<"vceqz", "U.", "hQh">;
1658 def CMGEH : SInst<"vcgez", "U.", "hQh">;
1659 def CMGTH : SInst<"vcgtz", "U.", "hQh">;
1660 def CMLEH : SInst<"vclez", "U.", "hQh">;
1661 def CMLTH : SInst<"vcltz", "U.", "hQh">;
1662
1663 // Vector conversion
1664 def VCVT_F16 : SInst<"vcvt_f16", "F(.!)", "sUsQsQUs">;
1665 def VCVT_S16 : SInst<"vcvt_s16", "S.", "hQh">;
1666 def VCVT_U16 : SInst<"vcvt_u16", "U.", "hQh">;
1667 def VCVTA_S16 : SInst<"vcvta_s16", "S.", "hQh">;
1668 def VCVTA_U16 : SInst<"vcvta_u16", "U.", "hQh">;
1669 def VCVTM_S16 : SInst<"vcvtm_s16", "S.", "hQh">;
1670 def VCVTM_U16 : SInst<"vcvtm_u16", "U.", "hQh">;
1671 def VCVTN_S16 : SInst<"vcvtn_s16", "S.", "hQh">;
1672 def VCVTN_U16 : SInst<"vcvtn_u16", "U.", "hQh">;
1673 def VCVTP_S16 : SInst<"vcvtp_s16", "S.", "hQh">;
1674 def VCVTP_U16 : SInst<"vcvtp_u16", "U.", "hQh">;
1675
1676 // Vector rounding
1677 let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_DIRECTED_ROUNDING) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)" in {
1678 def FRINTZH : SInst<"vrnd", "..", "hQh">;
1679 def FRINTNH : SInst<"vrndn", "..", "hQh">;
1680 def FRINTAH : SInst<"vrnda", "..", "hQh">;
1681 def FRINTPH : SInst<"vrndp", "..", "hQh">;
1682 def FRINTMH : SInst<"vrndm", "..", "hQh">;
1683 def FRINTXH : SInst<"vrndx", "..", "hQh">;
1684 }
1685
1686 // Misc.
1687 def VABSH : SInst<"vabs", "..", "hQh">;
1688 def VNEGH : SOpInst<"vneg", "..", "hQh", OP_NEG>;
1689 def VRECPEH : SInst<"vrecpe", "..", "hQh">;
1690 def FRSQRTEH : SInst<"vrsqrte", "..", "hQh">;
1691
1692 // ARMv8.2-A FP16 two-operands vector intrinsics.
1693
1694 // Misc.
1695 def VADDH : SOpInst<"vadd", "...", "hQh", OP_ADD>;
1696 def VABDH : SInst<"vabd", "...", "hQh">;
1697 def VSUBH : SOpInst<"vsub", "...", "hQh", OP_SUB>;
1698
1699 // Comparison
1700 let InstName = "vacge" in {
1701 def VCAGEH : SInst<"vcage", "U..", "hQh">;
1702 def VCALEH : SInst<"vcale", "U..", "hQh">;
1703 }
1704 let InstName = "vacgt" in {
1705 def VCAGTH : SInst<"vcagt", "U..", "hQh">;
1706 def VCALTH : SInst<"vcalt", "U..", "hQh">;
1707 }
1708 def VCEQH : SOpInst<"vceq", "U..", "hQh", OP_EQ>;
1709 def VCGEH : SOpInst<"vcge", "U..", "hQh", OP_GE>;
1710 def VCGTH : SOpInst<"vcgt", "U..", "hQh", OP_GT>;
1711 let InstName = "vcge" in
1712 def VCLEH : SOpInst<"vcle", "U..", "hQh", OP_LE>;
1713 let InstName = "vcgt" in
1714 def VCLTH : SOpInst<"vclt", "U..", "hQh", OP_LT>;
1715
1716 // Vector conversion
1717 let isVCVT_N = 1 in {
1718 def VCVT_N_F16 : SInst<"vcvt_n_f16", "F(.!)I", "sUsQsQUs">;
1719 def VCVT_N_S16 : SInst<"vcvt_n_s16", "S.I", "hQh">;
1720 def VCVT_N_U16 : SInst<"vcvt_n_u16", "U.I", "hQh">;
1721 }
1722
1723 // Max/Min
1724 def VMAXH : SInst<"vmax", "...", "hQh">;
1725 def VMINH : SInst<"vmin", "...", "hQh">;
1726 let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_NUMERIC_MAXMIN) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)" in {
1727 def FMAXNMH : SInst<"vmaxnm", "...", "hQh">;
1728 def FMINNMH : SInst<"vminnm", "...", "hQh">;
1729 }
1730
1731 // Multiplication/Division
1732 def VMULH : SOpInst<"vmul", "...", "hQh", OP_MUL>;
1733
1734 // Pairwise addition
1735 def VPADDH : SInst<"vpadd", "...", "h">;
1736
1737 // Pairwise Max/Min
1738 def VPMAXH : SInst<"vpmax", "...", "h">;
1739 def VPMINH : SInst<"vpmin", "...", "h">;
1740
1741 // Reciprocal/Sqrt
1742 def VRECPSH : SInst<"vrecps", "...", "hQh">;
1743 def VRSQRTSH : SInst<"vrsqrts", "...", "hQh">;
1744
1745 // ARMv8.2-A FP16 three-operands vector intrinsics.
1746
1747 // Vector fused multiply-add operations
1748 def VFMAH : SInst<"vfma", "....", "hQh">;
1749 def VFMSH : SOpInst<"vfms", "....", "hQh", OP_FMLS>;
1750
1751 // ARMv8.2-A FP16 lane vector intrinsics.
1752
1753 // Mul lane
1754 def VMUL_LANEH : IOpInst<"vmul_lane", "..qI", "hQh", OP_MUL_LN>;
1755 def VMUL_NH : IOpInst<"vmul_n", "..1", "hQh", OP_MUL_N>;
1756
1757 // Data processing intrinsics - section 5
1758
1759 // Logical operations
1760 let isHiddenLInst = 1 in
1761 def VBSLH : SInst<"vbsl", ".U..", "hQh">;
1762
1763 // Transposition operations
1764 def VZIPH : WInst<"vzip", "2..", "hQh">;
1765 def VUZPH : WInst<"vuzp", "2..", "hQh">;
1766 def VTRNH : WInst<"vtrn", "2..", "hQh">;
1767
1768
1769 let ArchGuard = "!defined(__aarch64__)" in {
1770 // Set all lanes to same value.
1771 // Already implemented prior to ARMv8.2-A.
1772 def VMOV_NH : WOpInst<"vmov_n", ".1", "hQh", OP_DUP>;
1773 def VDUP_NH : WOpInst<"vdup_n", ".1", "hQh", OP_DUP>;
1774 def VDUP_LANE1H : WOpInst<"vdup_lane", ".qI", "hQh", OP_DUP_LN>;
1775 }
1776
1777 // Vector Extract
1778 def VEXTH : WInst<"vext", "...I", "hQh">;
1779
1780 // Reverse vector elements
1781 def VREV64H : WOpInst<"vrev64", "..", "hQh", OP_REV64>;
1782}
1783
1784// ARMv8.2-A FP16 vector intrinsics for A64 only.
1785let ArchGuard = "defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(__aarch64__)" in {
1786
1787 // Vector rounding
1788 def FRINTIH : SInst<"vrndi", "..", "hQh">;
1789
1790 // Misc.
1791 def FSQRTH : SInst<"vsqrt", "..", "hQh">;
1792
1793 // Multiplication/Division
1794 def MULXH : SInst<"vmulx", "...", "hQh">;
1795 def FDIVH : IOpInst<"vdiv", "...", "hQh", OP_DIV>;
1796
1797 // Pairwise addition
1798 def VPADDH1 : SInst<"vpadd", "...", "Qh">;
1799
1800 // Pairwise Max/Min
1801 def VPMAXH1 : SInst<"vpmax", "...", "Qh">;
1802 def VPMINH1 : SInst<"vpmin", "...", "Qh">;
1803
1804 // Pairwise MaxNum/MinNum
1805 def FMAXNMPH : SInst<"vpmaxnm", "...", "hQh">;
1806 def FMINNMPH : SInst<"vpminnm", "...", "hQh">;
1807
1808 // ARMv8.2-A FP16 lane vector intrinsics.
1809
1810 // FMA lane
1811 def VFMA_LANEH : IInst<"vfma_lane", "...qI", "hQh">;
1812 def VFMA_LANEQH : IInst<"vfma_laneq", "...QI", "hQh"> {
1813 let isLaneQ = 1;
1814 }
1815
1816 // FMA lane with scalar argument
1817 def FMLA_NH : SOpInst<"vfma_n", "...1", "hQh", OP_FMLA_N>;
1818 // Scalar floating point fused multiply-add (scalar, by element)
1819 def SCALAR_FMLA_LANEH : IInst<"vfma_lane", "111.I", "Sh">;
1820 def SCALAR_FMLA_LANEQH : IInst<"vfma_laneq", "111QI", "Sh"> {
1821 let isLaneQ = 1;
1822 }
1823
1824 // FMS lane
1825 def VFMS_LANEH : IOpInst<"vfms_lane", "...qI", "hQh", OP_FMS_LN>;
1826 def VFMS_LANEQH : IOpInst<"vfms_laneq", "...QI", "hQh", OP_FMS_LNQ> {
1827 let isLaneQ = 1;
1828 }
1829 // FMS lane with scalar argument
1830 def FMLS_NH : SOpInst<"vfms_n", "...1", "hQh", OP_FMLS_N>;
1831 // Scalar floating foint fused multiply-subtract (scalar, by element)
1832 def SCALAR_FMLS_LANEH : IOpInst<"vfms_lane", "111.I", "Sh", OP_FMS_LN>;
1833 def SCALAR_FMLS_LANEQH : IOpInst<"vfms_laneq", "111QI", "Sh", OP_FMS_LNQ> {
1834 let isLaneQ = 1;
1835 }
1836
1837 // Mul lane
1838 def VMUL_LANEQH : IOpInst<"vmul_laneq", "..QI", "hQh", OP_MUL_LN> {
1839 let isLaneQ = 1;
1840 }
1841 // Scalar floating point multiply (scalar, by element)
1842 def SCALAR_FMUL_LANEH : IOpInst<"vmul_lane", "11.I", "Sh", OP_SCALAR_MUL_LN>;
1843 def SCALAR_FMUL_LANEQH : IOpInst<"vmul_laneq", "11QI", "Sh", OP_SCALAR_MUL_LN> {
1844 let isLaneQ = 1;
1845 }
1846
1847 // Mulx lane
1848 def VMULX_LANEH : IOpInst<"vmulx_lane", "..qI", "hQh", OP_MULX_LN>;
1849 def VMULX_LANEQH : IOpInst<"vmulx_laneq", "..QI", "hQh", OP_MULX_LN> {
1850 let isLaneQ = 1;
1851 }
1852 def VMULX_NH : IOpInst<"vmulx_n", "..1", "hQh", OP_MULX_N>;
1853 // Scalar floating point mulx (scalar, by element)
1854 def SCALAR_FMULX_LANEH : IInst<"vmulx_lane", "11.I", "Sh">;
1855 def SCALAR_FMULX_LANEQH : IInst<"vmulx_laneq", "11QI", "Sh"> {
1856 let isLaneQ = 1;
1857 }
1858
1859 // ARMv8.2-A FP16 reduction vector intrinsics.
1860 def VMAXVH : SInst<"vmaxv", "1.", "hQh">;
1861 def VMINVH : SInst<"vminv", "1.", "hQh">;
1862 def FMAXNMVH : SInst<"vmaxnmv", "1.", "hQh">;
1863 def FMINNMVH : SInst<"vminnmv", "1.", "hQh">;
1864
1865 // Permutation
1866 def VTRN1H : SOpInst<"vtrn1", "...", "hQh", OP_TRN1>;
1867 def VZIP1H : SOpInst<"vzip1", "...", "hQh", OP_ZIP1>;
1868 def VUZP1H : SOpInst<"vuzp1", "...", "hQh", OP_UZP1>;
1869 def VTRN2H : SOpInst<"vtrn2", "...", "hQh", OP_TRN2>;
1870 def VZIP2H : SOpInst<"vzip2", "...", "hQh", OP_ZIP2>;
1871 def VUZP2H : SOpInst<"vuzp2", "...", "hQh", OP_UZP2>;
1872
1873 def SCALAR_VDUP_LANEH : IInst<"vdup_lane", "1.I", "Sh">;
1874 def SCALAR_VDUP_LANEQH : IInst<"vdup_laneq", "1QI", "Sh"> {
1875 let isLaneQ = 1;
1876 }
1877}
1878
1879// v8.2-A dot product instructions.
1880let ArchGuard = "defined(__ARM_FEATURE_DOTPROD)" in {
1881 def DOT : SInst<"vdot", "..(<<)(<<)", "iQiUiQUi">;
1882 def DOT_LANE : SOpInst<"vdot_lane", "..(<<)(<<q)I", "iUiQiQUi", OP_DOT_LN>;
1883}
1884let ArchGuard = "defined(__ARM_FEATURE_DOTPROD) && defined(__aarch64__)" in {
1885 // Variants indexing into a 128-bit vector are A64 only.
1886 def UDOT_LANEQ : SOpInst<"vdot_laneq", "..(<<)(<<Q)I", "iUiQiQUi", OP_DOT_LNQ> {
1887 let isLaneQ = 1;
1888 }
1889}
1890
1891// v8.2-A FP16 fused multiply-add long instructions.
1892let ArchGuard = "defined(__ARM_FEATURE_FP16_FML) && defined(__aarch64__)" in {
1893 def VFMLAL_LOW : SInst<"vfmlal_low", ">>..", "hQh">;
1894 def VFMLSL_LOW : SInst<"vfmlsl_low", ">>..", "hQh">;
1895 def VFMLAL_HIGH : SInst<"vfmlal_high", ">>..", "hQh">;
1896 def VFMLSL_HIGH : SInst<"vfmlsl_high", ">>..", "hQh">;
1897
1898 def VFMLAL_LANE_LOW : SOpInst<"vfmlal_lane_low", "(F>)(F>)F(Fq)I", "hQh", OP_FMLAL_LN>;
1899 def VFMLSL_LANE_LOW : SOpInst<"vfmlsl_lane_low", "(F>)(F>)F(Fq)I", "hQh", OP_FMLSL_LN>;
1900 def VFMLAL_LANE_HIGH : SOpInst<"vfmlal_lane_high", "(F>)(F>)F(Fq)I", "hQh", OP_FMLAL_LN_Hi>;
1901 def VFMLSL_LANE_HIGH : SOpInst<"vfmlsl_lane_high", "(F>)(F>)F(Fq)I", "hQh", OP_FMLSL_LN_Hi>;
1902
1903 def VFMLAL_LANEQ_LOW : SOpInst<"vfmlal_laneq_low", "(F>)(F>)F(FQ)I", "hQh", OP_FMLAL_LN> {
1904 let isLaneQ = 1;
1905 }
1906 def VFMLSL_LANEQ_LOW : SOpInst<"vfmlsl_laneq_low", "(F>)(F>)F(FQ)I", "hQh", OP_FMLSL_LN> {
1907 let isLaneQ = 1;
1908 }
1909 def VFMLAL_LANEQ_HIGH : SOpInst<"vfmlal_laneq_high", "(F>)(F>)F(FQ)I", "hQh", OP_FMLAL_LN_Hi> {
1910 let isLaneQ = 1;
1911 }
1912 def VFMLSL_LANEQ_HIGH : SOpInst<"vfmlsl_laneq_high", "(F>)(F>)F(FQ)I", "hQh", OP_FMLSL_LN_Hi> {
1913 let isLaneQ = 1;
1914 }
1915}
1916
1917let ArchGuard = "defined(__ARM_FEATURE_MATMUL_INT8)" in {
1918 def VMMLA : SInst<"vmmla", "..(<<)(<<)", "QUiQi">;
1919 def VUSMMLA : SInst<"vusmmla", "..(<<U)(<<)", "Qi">;
1920
1921 def VUSDOT : SInst<"vusdot", "..(<<U)(<<)", "iQi">;
1922
1923 def VUSDOT_LANE : SOpInst<"vusdot_lane", "..(<<U)(<<q)I", "iQi", OP_USDOT_LN>;
1924 def VSUDOT_LANE : SOpInst<"vsudot_lane", "..(<<)(<<qU)I", "iQi", OP_SUDOT_LN>;
1925
1926 let ArchGuard = "defined(__aarch64__)" in {
1927 let isLaneQ = 1 in {
1928 def VUSDOT_LANEQ : SOpInst<"vusdot_laneq", "..(<<U)(<<Q)I", "iQi", OP_USDOT_LNQ>;
1929 def VSUDOT_LANEQ : SOpInst<"vsudot_laneq", "..(<<)(<<QU)I", "iQi", OP_SUDOT_LNQ>;
1930 }
1931 }
1932}
1933
1934let ArchGuard = "defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC)" in {
1935 def VDOT_BF : SInst<"vbfdot", "..BB", "fQf">;
1936 def VDOT_LANE_BF : SOpInst<"vbfdot_lane", "..B(Bq)I", "fQf", OP_BFDOT_LN>;
1937 def VDOT_LANEQ_BF : SOpInst<"vbfdot_laneq", "..B(BQ)I", "fQf", OP_BFDOT_LNQ> {
1938 let isLaneQ = 1;
1939 }
1940
1941 def VFMMLA_BF : SInst<"vbfmmla", "..BB", "Qf">;
1942
1943 def VFMLALB_BF : SInst<"vbfmlalb", "..BB", "Qf">;
1944 def VFMLALT_BF : SInst<"vbfmlalt", "..BB", "Qf">;
1945
1946 def VFMLALB_LANE_BF : SOpInst<"vbfmlalb_lane", "..B(Bq)I", "Qf", OP_BFMLALB_LN>;
1947 def VFMLALB_LANEQ_BF : SOpInst<"vbfmlalb_laneq", "..B(BQ)I", "Qf", OP_BFMLALB_LN>;
1948
1949 def VFMLALT_LANE_BF : SOpInst<"vbfmlalt_lane", "..B(Bq)I", "Qf", OP_BFMLALT_LN>;
1950 def VFMLALT_LANEQ_BF : SOpInst<"vbfmlalt_laneq", "..B(BQ)I", "Qf", OP_BFMLALT_LN>;
1951}
1952
1953multiclass VCMLA_ROTS<string type, string lanety, string laneqty> {
1954 foreach ROT = ["", "_rot90", "_rot180", "_rot270" ] in {
1955 def : SInst<"vcmla" # ROT, "....", type # "Q" # type>;
1956
1957 // vcmla{ROT}_lane
1958 def : SOpInst<"vcmla" # ROT # "_lane", "...qI", type, Op<(call "vcmla" # ROT, $p0, $p1,
1959 (bitcast $p0, (dup_typed lanety , (call "vget_lane", (bitcast lanety, $p2), $p3))))>>;
1960
1961 // vcmlaq{ROT}_lane
1962 def : SOpInst<"vcmla" # ROT # "_lane", "...qI", "Q" # type, Op<(call "vcmla" # ROT, $p0, $p1,
1963 (bitcast $p0, (dup_typed laneqty , (call "vget_lane", (bitcast lanety, $p2), $p3))))>>;
1964
1965 let isLaneQ = 1 in {
1966 // vcmla{ROT}_laneq
1967 def : SOpInst<"vcmla" # ROT # "_laneq", "...QI", type, Op<(call "vcmla" # ROT, $p0, $p1,
1968 (bitcast $p0, (dup_typed lanety, (call "vget_lane", (bitcast laneqty, $p2), $p3))))>>;
1969
1970 // vcmlaq{ROT}_laneq
1971 def : SOpInst<"vcmla" # ROT # "_laneq", "...QI", "Q" # type, Op<(call "vcmla" # ROT, $p0, $p1,
1972 (bitcast $p0, (dup_typed laneqty , (call "vget_lane", (bitcast laneqty, $p2), $p3))))>>;
1973 }
1974 }
1975}
1976
1977// v8.3-A Vector complex addition intrinsics
1978let ArchGuard = "defined(__ARM_FEATURE_COMPLEX) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)" in {
1979 def VCADD_ROT90_FP16 : SInst<"vcadd_rot90", "...", "h">;
1980 def VCADD_ROT270_FP16 : SInst<"vcadd_rot270", "...", "h">;
1981 def VCADDQ_ROT90_FP16 : SInst<"vcaddq_rot90", "QQQ", "h">;
1982 def VCADDQ_ROT270_FP16 : SInst<"vcaddq_rot270", "QQQ", "h">;
1983
1984 defm VCMLA_FP16 : VCMLA_ROTS<"h", "uint32x2_t", "uint32x4_t">;
1985}
1986let ArchGuard = "defined(__ARM_FEATURE_COMPLEX)" in {
1987 def VCADD_ROT90 : SInst<"vcadd_rot90", "...", "f">;
1988 def VCADD_ROT270 : SInst<"vcadd_rot270", "...", "f">;
1989 def VCADDQ_ROT90 : SInst<"vcaddq_rot90", "QQQ", "f">;
1990 def VCADDQ_ROT270 : SInst<"vcaddq_rot270", "QQQ", "f">;
1991
1992 defm VCMLA_F32 : VCMLA_ROTS<"f", "uint64x1_t", "uint64x2_t">;
1993}
1994let ArchGuard = "defined(__ARM_FEATURE_COMPLEX) && defined(__aarch64__)" in {
1995 def VCADDQ_ROT90_FP64 : SInst<"vcaddq_rot90", "QQQ", "d">;
1996 def VCADDQ_ROT270_FP64 : SInst<"vcaddq_rot270", "QQQ", "d">;
1997
1998 defm VCMLA_FP64 : VCMLA_ROTS<"d", "uint64x2_t", "uint64x2_t">;
1999}
2000
2001// V8.2-A BFloat intrinsics
2002let ArchGuard = "defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC)" in {
2003 def VCREATE_BF : NoTestOpInst<"vcreate", ".(IU>)", "b", OP_CAST> {
2004 let BigEndianSafe = 1;
2005 }
2006
2007 def VDUP_N_BF : WOpInst<"vdup_n", ".1", "bQb", OP_DUP>;
2008
2009 def VDUP_LANE_BF : WOpInst<"vdup_lane", ".qI", "bQb", OP_DUP_LN>;
2010 def VDUP_LANEQ_BF: WOpInst<"vdup_laneq", ".QI", "bQb", OP_DUP_LN> {
2011 let isLaneQ = 1;
2012 }
2013
2014 def VCOMBINE_BF : NoTestOpInst<"vcombine", "Q..", "b", OP_CONC>;
2015
2016 def VGET_HIGH_BF : NoTestOpInst<"vget_high", ".Q", "b", OP_HI>;
2017 def VGET_LOW_BF : NoTestOpInst<"vget_low", ".Q", "b", OP_LO>;
2018
2019 def VGET_LANE_BF : IInst<"vget_lane", "1.I", "bQb">;
2020 def VSET_LANE_BF : IInst<"vset_lane", ".1.I", "bQb">;
2021 def SCALAR_VDUP_LANE_BF : IInst<"vdup_lane", "1.I", "Sb">;
2022 def SCALAR_VDUP_LANEQ_BF : IInst<"vdup_laneq", "1QI", "Sb"> {
2023 let isLaneQ = 1;
2024 }
2025
2026 def VLD1_BF : WInst<"vld1", ".(c*!)", "bQb">;
2027 def VLD2_BF : WInst<"vld2", "2(c*!)", "bQb">;
2028 def VLD3_BF : WInst<"vld3", "3(c*!)", "bQb">;
2029 def VLD4_BF : WInst<"vld4", "4(c*!)", "bQb">;
2030
2031 def VST1_BF : WInst<"vst1", "v*(.!)", "bQb">;
2032 def VST2_BF : WInst<"vst2", "v*(2!)", "bQb">;
2033 def VST3_BF : WInst<"vst3", "v*(3!)", "bQb">;
2034 def VST4_BF : WInst<"vst4", "v*(4!)", "bQb">;
2035
2036 def VLD1_X2_BF : WInst<"vld1_x2", "2(c*!)", "bQb">;
2037 def VLD1_X3_BF : WInst<"vld1_x3", "3(c*!)", "bQb">;
2038 def VLD1_X4_BF : WInst<"vld1_x4", "4(c*!)", "bQb">;
2039
2040 def VST1_X2_BF : WInst<"vst1_x2", "v*(2!)", "bQb">;
2041 def VST1_X3_BF : WInst<"vst1_x3", "v*(3!)", "bQb">;
2042 def VST1_X4_BF : WInst<"vst1_x4", "v*(4!)", "bQb">;
2043
2044 def VLD1_LANE_BF : WInst<"vld1_lane", ".(c*!).I", "bQb">;
2045 def VLD2_LANE_BF : WInst<"vld2_lane", "2(c*!)2I", "bQb">;
2046 def VLD3_LANE_BF : WInst<"vld3_lane", "3(c*!)3I", "bQb">;
2047 def VLD4_LANE_BF : WInst<"vld4_lane", "4(c*!)4I", "bQb">;
2048 def VST1_LANE_BF : WInst<"vst1_lane", "v*(.!)I", "bQb">;
2049 def VST2_LANE_BF : WInst<"vst2_lane", "v*(2!)I", "bQb">;
2050 def VST3_LANE_BF : WInst<"vst3_lane", "v*(3!)I", "bQb">;
2051 def VST4_LANE_BF : WInst<"vst4_lane", "v*(4!)I", "bQb">;
2052
2053 def VLD1_DUP_BF : WInst<"vld1_dup", ".(c*!)", "bQb">;
2054 def VLD2_DUP_BF : WInst<"vld2_dup", "2(c*!)", "bQb">;
2055 def VLD3_DUP_BF : WInst<"vld3_dup", "3(c*!)", "bQb">;
2056 def VLD4_DUP_BF : WInst<"vld4_dup", "4(c*!)", "bQb">;
2057
2058 def VCVT_F32_BF16 : SOpInst<"vcvt_f32_bf16", "(F>)(Bq!)", "Qb", OP_VCVT_F32_BF16>;
2059 def VCVT_LOW_F32_BF16 : SOpInst<"vcvt_low_f32", "(F>)(BQ!)", "Qb", OP_VCVT_F32_BF16_LO>;
2060 def VCVT_HIGH_F32_BF16 : SOpInst<"vcvt_high_f32", "(F>)(BQ!)", "Qb", OP_VCVT_F32_BF16_HI>;
2061
2062 def SCALAR_CVT_BF16_F32 : SInst<"vcvth_bf16", "(1B)1", "f">;
2063 def SCALAR_CVT_F32_BF16 : SOpInst<"vcvtah_f32", "(1F>)(1!)", "b", OP_CVT_F32_BF16>;
2064}
2065
2066let ArchGuard = "defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC) && !defined(__aarch64__)" in {
2067 def VCVT_BF16_F32_A32_INTERNAL : WInst<"__a32_vcvt_bf16", "BQ", "f">;
2068 def VCVT_BF16_F32_A32 : SOpInst<"vcvt_bf16", "BQ", "f", OP_VCVT_BF16_F32_A32>;
2069 def VCVT_LOW_BF16_F32_A32 : SOpInst<"vcvt_low_bf16", "BQ", "Qf", OP_VCVT_BF16_F32_LO_A32>;
2070 def VCVT_HIGH_BF16_F32_A32 : SOpInst<"vcvt_high_bf16", "BBQ", "Qf", OP_VCVT_BF16_F32_HI_A32>;
2071}
2072
2073let ArchGuard = "defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC) && defined(__aarch64__)" in {
2074 def VCVT_LOW_BF16_F32_A64_INTERNAL : WInst<"__a64_vcvtq_low_bf16", "BQ", "Hf">;
2075 def VCVT_LOW_BF16_F32_A64 : SOpInst<"vcvt_low_bf16", "BQ", "Qf", OP_VCVT_BF16_F32_LO_A64>;
2076 def VCVT_HIGH_BF16_F32_A64 : SInst<"vcvt_high_bf16", "BBQ", "Qf">;
2077 def VCVT_BF16_F32 : SOpInst<"vcvt_bf16", "BQ", "f", OP_VCVT_BF16_F32_A64>;
2078
2079 def COPY_LANE_BF16 : IOpInst<"vcopy_lane", "..I.I", "b", OP_COPY_LN>;
2080 def COPYQ_LANE_BF16 : IOpInst<"vcopy_lane", "..IqI", "Qb", OP_COPY_LN>;
2081 def COPY_LANEQ_BF16 : IOpInst<"vcopy_laneq", "..IQI", "b", OP_COPY_LN>;
2082 def COPYQ_LANEQ_BF16 : IOpInst<"vcopy_laneq", "..I.I", "Qb", OP_COPY_LN>;
2083}
2084
2085let ArchGuard = "defined(__ARM_FEATURE_BF16) && !defined(__aarch64__)" in {
2086 let BigEndianSafe = 1 in {
2087 defm VREINTERPRET_BF : REINTERPRET_CROSS_TYPES<
2088 "csilUcUsUiUlhfPcPsPlQcQsQiQlQUcQUsQUiQUlQhQfQPcQPsQPl", "bQb">;
2089 }
2090}
2091
2092let ArchGuard = "defined(__ARM_FEATURE_BF16) && defined(__aarch64__)" in {
2093 let BigEndianSafe = 1 in {
2094 defm VVREINTERPRET_BF : REINTERPRET_CROSS_TYPES<
2095 "csilUcUsUiUlhfdPcPsPlQcQsQiQlQUcQUsQUiQUlQhQfQdQPcQPsQPlQPk", "bQb">;
2096 }
2097}
2098