1//===--- BuiltinsPTX.def - PTX Builtin function database ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the PTX-specific builtin function database. Users of
10// this file must define the BUILTIN macro to make use of this information.
11//
12//===----------------------------------------------------------------------===//
13
14// The format of this database matches clang/Basic/Builtins.def.
15
16#if defined(BUILTIN) && !defined(TARGET_BUILTIN)
17# define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE) BUILTIN(ID, TYPE, ATTRS)
18#endif
19
20#pragma push_macro("SM_70")
21#pragma push_macro("SM_72")
22#pragma push_macro("SM_75")
23#pragma push_macro("SM_80")
24#pragma push_macro("SM_86")
25#define SM_86 "sm_86"
26#define SM_80 "sm_80|" SM_86
27#define SM_75 "sm_75|" SM_80
28#define SM_72 "sm_72|" SM_75
29#define SM_70 "sm_70|" SM_72
30
31#pragma push_macro("SM_60")
32#define SM_60 "sm_60|sm_61|sm_62|" SM_70
33
34#pragma push_macro("PTX60")
35#pragma push_macro("PTX61")
36#pragma push_macro("PTX63")
37#pragma push_macro("PTX64")
38#pragma push_macro("PTX65")
39#pragma push_macro("PTX70")
40#pragma push_macro("PTX71")
41#pragma push_macro("PTX72")
42#define PTX72 "ptx72"
43#define PTX71 "ptx71|" PTX72
44#define PTX70 "ptx70|" PTX71
45#define PTX65 "ptx65|" PTX70
46#define PTX64 "ptx64|" PTX65
47#define PTX63 "ptx63|" PTX64
48#define PTX61 "ptx61|" PTX63
49#define PTX60 "ptx60|" PTX61
50
51#pragma push_macro("AND")
52#define AND(a, b) "(" a "),(" b ")"
53
54// Special Registers
55
56BUILTIN(__nvvm_read_ptx_sreg_tid_x, "i", "nc")
57BUILTIN(__nvvm_read_ptx_sreg_tid_y, "i", "nc")
58BUILTIN(__nvvm_read_ptx_sreg_tid_z, "i", "nc")
59BUILTIN(__nvvm_read_ptx_sreg_tid_w, "i", "nc")
60
61BUILTIN(__nvvm_read_ptx_sreg_ntid_x, "i", "nc")
62BUILTIN(__nvvm_read_ptx_sreg_ntid_y, "i", "nc")
63BUILTIN(__nvvm_read_ptx_sreg_ntid_z, "i", "nc")
64BUILTIN(__nvvm_read_ptx_sreg_ntid_w, "i", "nc")
65
66BUILTIN(__nvvm_read_ptx_sreg_ctaid_x, "i", "nc")
67BUILTIN(__nvvm_read_ptx_sreg_ctaid_y, "i", "nc")
68BUILTIN(__nvvm_read_ptx_sreg_ctaid_z, "i", "nc")
69BUILTIN(__nvvm_read_ptx_sreg_ctaid_w, "i", "nc")
70
71BUILTIN(__nvvm_read_ptx_sreg_nctaid_x, "i", "nc")
72BUILTIN(__nvvm_read_ptx_sreg_nctaid_y, "i", "nc")
73BUILTIN(__nvvm_read_ptx_sreg_nctaid_z, "i", "nc")
74BUILTIN(__nvvm_read_ptx_sreg_nctaid_w, "i", "nc")
75
76BUILTIN(__nvvm_read_ptx_sreg_laneid, "i", "nc")
77BUILTIN(__nvvm_read_ptx_sreg_warpid, "i", "nc")
78BUILTIN(__nvvm_read_ptx_sreg_nwarpid, "i", "nc")
79
80BUILTIN(__nvvm_read_ptx_sreg_smid, "i", "nc")
81BUILTIN(__nvvm_read_ptx_sreg_nsmid, "i", "nc")
82BUILTIN(__nvvm_read_ptx_sreg_gridid, "i", "nc")
83
84BUILTIN(__nvvm_read_ptx_sreg_lanemask_eq, "i", "nc")
85BUILTIN(__nvvm_read_ptx_sreg_lanemask_le, "i", "nc")
86BUILTIN(__nvvm_read_ptx_sreg_lanemask_lt, "i", "nc")
87BUILTIN(__nvvm_read_ptx_sreg_lanemask_ge, "i", "nc")
88BUILTIN(__nvvm_read_ptx_sreg_lanemask_gt, "i", "nc")
89
90BUILTIN(__nvvm_read_ptx_sreg_clock, "i", "n")
91BUILTIN(__nvvm_read_ptx_sreg_clock64, "LLi", "n")
92
93BUILTIN(__nvvm_read_ptx_sreg_pm0, "i", "n")
94BUILTIN(__nvvm_read_ptx_sreg_pm1, "i", "n")
95BUILTIN(__nvvm_read_ptx_sreg_pm2, "i", "n")
96BUILTIN(__nvvm_read_ptx_sreg_pm3, "i", "n")
97
98// MISC
99
100BUILTIN(__nvvm_prmt, "UiUiUiUi", "")
101
102// Min Max
103
104BUILTIN(__nvvm_fmax_ftz_f, "fff", "")
105BUILTIN(__nvvm_fmax_f, "fff", "")
106BUILTIN(__nvvm_fmin_ftz_f, "fff", "")
107BUILTIN(__nvvm_fmin_f, "fff", "")
108
109BUILTIN(__nvvm_fmax_d, "ddd", "")
110BUILTIN(__nvvm_fmin_d, "ddd", "")
111
112// Multiplication
113
114BUILTIN(__nvvm_mulhi_i, "iii", "")
115BUILTIN(__nvvm_mulhi_ui, "UiUiUi", "")
116BUILTIN(__nvvm_mulhi_ll, "LLiLLiLLi", "")
117BUILTIN(__nvvm_mulhi_ull, "ULLiULLiULLi", "")
118
119BUILTIN(__nvvm_mul_rn_ftz_f, "fff", "")
120BUILTIN(__nvvm_mul_rn_f, "fff", "")
121BUILTIN(__nvvm_mul_rz_ftz_f, "fff", "")
122BUILTIN(__nvvm_mul_rz_f, "fff", "")
123BUILTIN(__nvvm_mul_rm_ftz_f, "fff", "")
124BUILTIN(__nvvm_mul_rm_f, "fff", "")
125BUILTIN(__nvvm_mul_rp_ftz_f, "fff", "")
126BUILTIN(__nvvm_mul_rp_f, "fff", "")
127
128BUILTIN(__nvvm_mul_rn_d, "ddd", "")
129BUILTIN(__nvvm_mul_rz_d, "ddd", "")
130BUILTIN(__nvvm_mul_rm_d, "ddd", "")
131BUILTIN(__nvvm_mul_rp_d, "ddd", "")
132
133BUILTIN(__nvvm_mul24_i, "iii", "")
134BUILTIN(__nvvm_mul24_ui, "UiUiUi", "")
135
136// Div
137
138BUILTIN(__nvvm_div_approx_ftz_f, "fff", "")
139BUILTIN(__nvvm_div_approx_f, "fff", "")
140
141BUILTIN(__nvvm_div_rn_ftz_f, "fff", "")
142BUILTIN(__nvvm_div_rn_f, "fff", "")
143BUILTIN(__nvvm_div_rz_ftz_f, "fff", "")
144BUILTIN(__nvvm_div_rz_f, "fff", "")
145BUILTIN(__nvvm_div_rm_ftz_f, "fff", "")
146BUILTIN(__nvvm_div_rm_f, "fff", "")
147BUILTIN(__nvvm_div_rp_ftz_f, "fff", "")
148BUILTIN(__nvvm_div_rp_f, "fff", "")
149
150BUILTIN(__nvvm_div_rn_d, "ddd", "")
151BUILTIN(__nvvm_div_rz_d, "ddd", "")
152BUILTIN(__nvvm_div_rm_d, "ddd", "")
153BUILTIN(__nvvm_div_rp_d, "ddd", "")
154
155// Sad
156
157BUILTIN(__nvvm_sad_i, "iiii", "")
158BUILTIN(__nvvm_sad_ui, "UiUiUiUi", "")
159
160// Floor, Ceil
161
162BUILTIN(__nvvm_floor_ftz_f, "ff", "")
163BUILTIN(__nvvm_floor_f, "ff", "")
164BUILTIN(__nvvm_floor_d, "dd", "")
165
166BUILTIN(__nvvm_ceil_ftz_f, "ff", "")
167BUILTIN(__nvvm_ceil_f, "ff", "")
168BUILTIN(__nvvm_ceil_d, "dd", "")
169
170// Abs
171
172BUILTIN(__nvvm_fabs_ftz_f, "ff", "")
173BUILTIN(__nvvm_fabs_f, "ff", "")
174BUILTIN(__nvvm_fabs_d, "dd", "")
175
176// Round
177
178BUILTIN(__nvvm_round_ftz_f, "ff", "")
179BUILTIN(__nvvm_round_f, "ff", "")
180BUILTIN(__nvvm_round_d, "dd", "")
181
182// Trunc
183
184BUILTIN(__nvvm_trunc_ftz_f, "ff", "")
185BUILTIN(__nvvm_trunc_f, "ff", "")
186BUILTIN(__nvvm_trunc_d, "dd", "")
187
188// Saturate
189
190BUILTIN(__nvvm_saturate_ftz_f, "ff", "")
191BUILTIN(__nvvm_saturate_f, "ff", "")
192BUILTIN(__nvvm_saturate_d, "dd", "")
193
194// Exp2, Log2
195
196BUILTIN(__nvvm_ex2_approx_ftz_f, "ff", "")
197BUILTIN(__nvvm_ex2_approx_f, "ff", "")
198BUILTIN(__nvvm_ex2_approx_d, "dd", "")
199
200BUILTIN(__nvvm_lg2_approx_ftz_f, "ff", "")
201BUILTIN(__nvvm_lg2_approx_f, "ff", "")
202BUILTIN(__nvvm_lg2_approx_d, "dd", "")
203
204// Sin, Cos
205
206BUILTIN(__nvvm_sin_approx_ftz_f, "ff", "")
207BUILTIN(__nvvm_sin_approx_f, "ff", "")
208
209BUILTIN(__nvvm_cos_approx_ftz_f, "ff", "")
210BUILTIN(__nvvm_cos_approx_f, "ff", "")
211
212// Fma
213
214BUILTIN(__nvvm_fma_rn_ftz_f, "ffff", "")
215BUILTIN(__nvvm_fma_rn_f, "ffff", "")
216BUILTIN(__nvvm_fma_rz_ftz_f, "ffff", "")
217BUILTIN(__nvvm_fma_rz_f, "ffff", "")
218BUILTIN(__nvvm_fma_rm_ftz_f, "ffff", "")
219BUILTIN(__nvvm_fma_rm_f, "ffff", "")
220BUILTIN(__nvvm_fma_rp_ftz_f, "ffff", "")
221BUILTIN(__nvvm_fma_rp_f, "ffff", "")
222BUILTIN(__nvvm_fma_rn_d, "dddd", "")
223BUILTIN(__nvvm_fma_rz_d, "dddd", "")
224BUILTIN(__nvvm_fma_rm_d, "dddd", "")
225BUILTIN(__nvvm_fma_rp_d, "dddd", "")
226
227// Rcp
228
229BUILTIN(__nvvm_rcp_rn_ftz_f, "ff", "")
230BUILTIN(__nvvm_rcp_rn_f, "ff", "")
231BUILTIN(__nvvm_rcp_rz_ftz_f, "ff", "")
232BUILTIN(__nvvm_rcp_rz_f, "ff", "")
233BUILTIN(__nvvm_rcp_rm_ftz_f, "ff", "")
234BUILTIN(__nvvm_rcp_rm_f, "ff", "")
235BUILTIN(__nvvm_rcp_rp_ftz_f, "ff", "")
236BUILTIN(__nvvm_rcp_rp_f, "ff", "")
237
238BUILTIN(__nvvm_rcp_rn_d, "dd", "")
239BUILTIN(__nvvm_rcp_rz_d, "dd", "")
240BUILTIN(__nvvm_rcp_rm_d, "dd", "")
241BUILTIN(__nvvm_rcp_rp_d, "dd", "")
242BUILTIN(__nvvm_rcp_approx_ftz_d, "dd", "")
243
244// Sqrt
245
246BUILTIN(__nvvm_sqrt_rn_ftz_f, "ff", "")
247BUILTIN(__nvvm_sqrt_rn_f, "ff", "")
248BUILTIN(__nvvm_sqrt_rz_ftz_f, "ff", "")
249BUILTIN(__nvvm_sqrt_rz_f, "ff", "")
250BUILTIN(__nvvm_sqrt_rm_ftz_f, "ff", "")
251BUILTIN(__nvvm_sqrt_rm_f, "ff", "")
252BUILTIN(__nvvm_sqrt_rp_ftz_f, "ff", "")
253BUILTIN(__nvvm_sqrt_rp_f, "ff", "")
254BUILTIN(__nvvm_sqrt_approx_ftz_f, "ff", "")
255BUILTIN(__nvvm_sqrt_approx_f, "ff", "")
256
257BUILTIN(__nvvm_sqrt_rn_d, "dd", "")
258BUILTIN(__nvvm_sqrt_rz_d, "dd", "")
259BUILTIN(__nvvm_sqrt_rm_d, "dd", "")
260BUILTIN(__nvvm_sqrt_rp_d, "dd", "")
261
262// Rsqrt
263
264BUILTIN(__nvvm_rsqrt_approx_ftz_f, "ff", "")
265BUILTIN(__nvvm_rsqrt_approx_f, "ff", "")
266BUILTIN(__nvvm_rsqrt_approx_d, "dd", "")
267
268// Add
269
270BUILTIN(__nvvm_add_rn_ftz_f, "fff", "")
271BUILTIN(__nvvm_add_rn_f, "fff", "")
272BUILTIN(__nvvm_add_rz_ftz_f, "fff", "")
273BUILTIN(__nvvm_add_rz_f, "fff", "")
274BUILTIN(__nvvm_add_rm_ftz_f, "fff", "")
275BUILTIN(__nvvm_add_rm_f, "fff", "")
276BUILTIN(__nvvm_add_rp_ftz_f, "fff", "")
277BUILTIN(__nvvm_add_rp_f, "fff", "")
278
279BUILTIN(__nvvm_add_rn_d, "ddd", "")
280BUILTIN(__nvvm_add_rz_d, "ddd", "")
281BUILTIN(__nvvm_add_rm_d, "ddd", "")
282BUILTIN(__nvvm_add_rp_d, "ddd", "")
283
284// Convert
285
286BUILTIN(__nvvm_d2f_rn_ftz, "fd", "")
287BUILTIN(__nvvm_d2f_rn, "fd", "")
288BUILTIN(__nvvm_d2f_rz_ftz, "fd", "")
289BUILTIN(__nvvm_d2f_rz, "fd", "")
290BUILTIN(__nvvm_d2f_rm_ftz, "fd", "")
291BUILTIN(__nvvm_d2f_rm, "fd", "")
292BUILTIN(__nvvm_d2f_rp_ftz, "fd", "")
293BUILTIN(__nvvm_d2f_rp, "fd", "")
294
295BUILTIN(__nvvm_d2i_rn, "id", "")
296BUILTIN(__nvvm_d2i_rz, "id", "")
297BUILTIN(__nvvm_d2i_rm, "id", "")
298BUILTIN(__nvvm_d2i_rp, "id", "")
299
300BUILTIN(__nvvm_d2ui_rn, "Uid", "")
301BUILTIN(__nvvm_d2ui_rz, "Uid", "")
302BUILTIN(__nvvm_d2ui_rm, "Uid", "")
303BUILTIN(__nvvm_d2ui_rp, "Uid", "")
304
305BUILTIN(__nvvm_i2d_rn, "di", "")
306BUILTIN(__nvvm_i2d_rz, "di", "")
307BUILTIN(__nvvm_i2d_rm, "di", "")
308BUILTIN(__nvvm_i2d_rp, "di", "")
309
310BUILTIN(__nvvm_ui2d_rn, "dUi", "")
311BUILTIN(__nvvm_ui2d_rz, "dUi", "")
312BUILTIN(__nvvm_ui2d_rm, "dUi", "")
313BUILTIN(__nvvm_ui2d_rp, "dUi", "")
314
315BUILTIN(__nvvm_f2i_rn_ftz, "if", "")
316BUILTIN(__nvvm_f2i_rn, "if", "")
317BUILTIN(__nvvm_f2i_rz_ftz, "if", "")
318BUILTIN(__nvvm_f2i_rz, "if", "")
319BUILTIN(__nvvm_f2i_rm_ftz, "if", "")
320BUILTIN(__nvvm_f2i_rm, "if", "")
321BUILTIN(__nvvm_f2i_rp_ftz, "if", "")
322BUILTIN(__nvvm_f2i_rp, "if", "")
323
324BUILTIN(__nvvm_f2ui_rn_ftz, "Uif", "")
325BUILTIN(__nvvm_f2ui_rn, "Uif", "")
326BUILTIN(__nvvm_f2ui_rz_ftz, "Uif", "")
327BUILTIN(__nvvm_f2ui_rz, "Uif", "")
328BUILTIN(__nvvm_f2ui_rm_ftz, "Uif", "")
329BUILTIN(__nvvm_f2ui_rm, "Uif", "")
330BUILTIN(__nvvm_f2ui_rp_ftz, "Uif", "")
331BUILTIN(__nvvm_f2ui_rp, "Uif", "")
332
333BUILTIN(__nvvm_i2f_rn, "fi", "")
334BUILTIN(__nvvm_i2f_rz, "fi", "")
335BUILTIN(__nvvm_i2f_rm, "fi", "")
336BUILTIN(__nvvm_i2f_rp, "fi", "")
337
338BUILTIN(__nvvm_ui2f_rn, "fUi", "")
339BUILTIN(__nvvm_ui2f_rz, "fUi", "")
340BUILTIN(__nvvm_ui2f_rm, "fUi", "")
341BUILTIN(__nvvm_ui2f_rp, "fUi", "")
342
343BUILTIN(__nvvm_lohi_i2d, "dii", "")
344
345BUILTIN(__nvvm_d2i_lo, "id", "")
346BUILTIN(__nvvm_d2i_hi, "id", "")
347
348BUILTIN(__nvvm_f2ll_rn_ftz, "LLif", "")
349BUILTIN(__nvvm_f2ll_rn, "LLif", "")
350BUILTIN(__nvvm_f2ll_rz_ftz, "LLif", "")
351BUILTIN(__nvvm_f2ll_rz, "LLif", "")
352BUILTIN(__nvvm_f2ll_rm_ftz, "LLif", "")
353BUILTIN(__nvvm_f2ll_rm, "LLif", "")
354BUILTIN(__nvvm_f2ll_rp_ftz, "LLif", "")
355BUILTIN(__nvvm_f2ll_rp, "LLif", "")
356
357BUILTIN(__nvvm_f2ull_rn_ftz, "ULLif", "")
358BUILTIN(__nvvm_f2ull_rn, "ULLif", "")
359BUILTIN(__nvvm_f2ull_rz_ftz, "ULLif", "")
360BUILTIN(__nvvm_f2ull_rz, "ULLif", "")
361BUILTIN(__nvvm_f2ull_rm_ftz, "ULLif", "")
362BUILTIN(__nvvm_f2ull_rm, "ULLif", "")
363BUILTIN(__nvvm_f2ull_rp_ftz, "ULLif", "")
364BUILTIN(__nvvm_f2ull_rp, "ULLif", "")
365
366BUILTIN(__nvvm_d2ll_rn, "LLid", "")
367BUILTIN(__nvvm_d2ll_rz, "LLid", "")
368BUILTIN(__nvvm_d2ll_rm, "LLid", "")
369BUILTIN(__nvvm_d2ll_rp, "LLid", "")
370
371BUILTIN(__nvvm_d2ull_rn, "ULLid", "")
372BUILTIN(__nvvm_d2ull_rz, "ULLid", "")
373BUILTIN(__nvvm_d2ull_rm, "ULLid", "")
374BUILTIN(__nvvm_d2ull_rp, "ULLid", "")
375
376BUILTIN(__nvvm_ll2f_rn, "fLLi", "")
377BUILTIN(__nvvm_ll2f_rz, "fLLi", "")
378BUILTIN(__nvvm_ll2f_rm, "fLLi", "")
379BUILTIN(__nvvm_ll2f_rp, "fLLi", "")
380
381BUILTIN(__nvvm_ull2f_rn, "fULLi", "")
382BUILTIN(__nvvm_ull2f_rz, "fULLi", "")
383BUILTIN(__nvvm_ull2f_rm, "fULLi", "")
384BUILTIN(__nvvm_ull2f_rp, "fULLi", "")
385
386BUILTIN(__nvvm_ll2d_rn, "dLLi", "")
387BUILTIN(__nvvm_ll2d_rz, "dLLi", "")
388BUILTIN(__nvvm_ll2d_rm, "dLLi", "")
389BUILTIN(__nvvm_ll2d_rp, "dLLi", "")
390
391BUILTIN(__nvvm_ull2d_rn, "dULLi", "")
392BUILTIN(__nvvm_ull2d_rz, "dULLi", "")
393BUILTIN(__nvvm_ull2d_rm, "dULLi", "")
394BUILTIN(__nvvm_ull2d_rp, "dULLi", "")
395
396BUILTIN(__nvvm_f2h_rn_ftz, "Usf", "")
397BUILTIN(__nvvm_f2h_rn, "Usf", "")
398
399// Bitcast
400
401BUILTIN(__nvvm_bitcast_f2i, "if", "")
402BUILTIN(__nvvm_bitcast_i2f, "fi", "")
403
404BUILTIN(__nvvm_bitcast_ll2d, "dLLi", "")
405BUILTIN(__nvvm_bitcast_d2ll, "LLid", "")
406
407// FNS
408TARGET_BUILTIN(__nvvm_fns, "UiUiUii", "n", PTX60)
409
410// Sync
411
412BUILTIN(__syncthreads, "v", "")
413BUILTIN(__nvvm_bar0_popc, "ii", "")
414BUILTIN(__nvvm_bar0_and, "ii", "")
415BUILTIN(__nvvm_bar0_or, "ii", "")
416BUILTIN(__nvvm_bar_sync, "vi", "n")
417TARGET_BUILTIN(__nvvm_bar_warp_sync, "vUi", "n", PTX60)
418TARGET_BUILTIN(__nvvm_barrier_sync, "vUi", "n", PTX60)
419TARGET_BUILTIN(__nvvm_barrier_sync_cnt, "vUiUi", "n", PTX60)
420
421// Shuffle
422
423BUILTIN(__nvvm_shfl_down_i32, "iiii", "")
424BUILTIN(__nvvm_shfl_down_f32, "ffii", "")
425BUILTIN(__nvvm_shfl_up_i32, "iiii", "")
426BUILTIN(__nvvm_shfl_up_f32, "ffii", "")
427BUILTIN(__nvvm_shfl_bfly_i32, "iiii", "")
428BUILTIN(__nvvm_shfl_bfly_f32, "ffii", "")
429BUILTIN(__nvvm_shfl_idx_i32, "iiii", "")
430BUILTIN(__nvvm_shfl_idx_f32, "ffii", "")
431
432TARGET_BUILTIN(__nvvm_shfl_sync_down_i32, "iUiiii", "", PTX60)
433TARGET_BUILTIN(__nvvm_shfl_sync_down_f32, "fUifii", "", PTX60)
434TARGET_BUILTIN(__nvvm_shfl_sync_up_i32, "iUiiii", "", PTX60)
435TARGET_BUILTIN(__nvvm_shfl_sync_up_f32, "fUifii", "", PTX60)
436TARGET_BUILTIN(__nvvm_shfl_sync_bfly_i32, "iUiiii", "", PTX60)
437TARGET_BUILTIN(__nvvm_shfl_sync_bfly_f32, "fUifii", "", PTX60)
438TARGET_BUILTIN(__nvvm_shfl_sync_idx_i32, "iUiiii", "", PTX60)
439TARGET_BUILTIN(__nvvm_shfl_sync_idx_f32, "fUifii", "", PTX60)
440
441// Vote
442BUILTIN(__nvvm_vote_all, "bb", "")
443BUILTIN(__nvvm_vote_any, "bb", "")
444BUILTIN(__nvvm_vote_uni, "bb", "")
445BUILTIN(__nvvm_vote_ballot, "Uib", "")
446
447TARGET_BUILTIN(__nvvm_vote_all_sync, "bUib", "", PTX60)
448TARGET_BUILTIN(__nvvm_vote_any_sync, "bUib", "", PTX60)
449TARGET_BUILTIN(__nvvm_vote_uni_sync, "bUib", "", PTX60)
450TARGET_BUILTIN(__nvvm_vote_ballot_sync, "UiUib", "", PTX60)
451
452// Match
453TARGET_BUILTIN(__nvvm_match_any_sync_i32, "UiUiUi", "", PTX60)
454TARGET_BUILTIN(__nvvm_match_any_sync_i64, "WiUiWi", "", PTX60)
455// These return a pair {value, predicate}, which requires custom lowering.
456TARGET_BUILTIN(__nvvm_match_all_sync_i32p, "UiUiUii*", "", PTX60)
457TARGET_BUILTIN(__nvvm_match_all_sync_i64p, "WiUiWii*", "", PTX60)
458
459// Membar
460
461BUILTIN(__nvvm_membar_cta, "v", "")
462BUILTIN(__nvvm_membar_gl, "v", "")
463BUILTIN(__nvvm_membar_sys, "v", "")
464
465// Memcpy, Memset
466
467BUILTIN(__nvvm_memcpy, "vUc*Uc*zi","")
468BUILTIN(__nvvm_memset, "vUc*Uczi","")
469
470// Image
471
472BUILTIN(__builtin_ptx_read_image2Dfi_, "V4fiiii", "")
473BUILTIN(__builtin_ptx_read_image2Dff_, "V4fiiff", "")
474BUILTIN(__builtin_ptx_read_image2Dii_, "V4iiiii", "")
475BUILTIN(__builtin_ptx_read_image2Dif_, "V4iiiff", "")
476
477BUILTIN(__builtin_ptx_read_image3Dfi_, "V4fiiiiii", "")
478BUILTIN(__builtin_ptx_read_image3Dff_, "V4fiiffff", "")
479BUILTIN(__builtin_ptx_read_image3Dii_, "V4iiiiiii", "")
480BUILTIN(__builtin_ptx_read_image3Dif_, "V4iiiffff", "")
481
482BUILTIN(__builtin_ptx_write_image2Df_, "viiiffff", "")
483BUILTIN(__builtin_ptx_write_image2Di_, "viiiiiii", "")
484BUILTIN(__builtin_ptx_write_image2Dui_, "viiiUiUiUiUi", "")
485BUILTIN(__builtin_ptx_get_image_depthi_, "ii", "")
486BUILTIN(__builtin_ptx_get_image_heighti_, "ii", "")
487BUILTIN(__builtin_ptx_get_image_widthi_, "ii", "")
488BUILTIN(__builtin_ptx_get_image_channel_data_typei_, "ii", "")
489BUILTIN(__builtin_ptx_get_image_channel_orderi_, "ii", "")
490
491// Atomic
492//
493// We need the atom intrinsics because
494// - they are used in converging analysis
495// - they are used in address space analysis and optimization
496// So it does not hurt to expose them as builtins.
497//
498BUILTIN(__nvvm_atom_add_gen_i, "iiD*i", "n")
499TARGET_BUILTIN(__nvvm_atom_cta_add_gen_i, "iiD*i", "n", SM_60)
500TARGET_BUILTIN(__nvvm_atom_sys_add_gen_i, "iiD*i", "n", SM_60)
501BUILTIN(__nvvm_atom_add_gen_l, "LiLiD*Li", "n")
502TARGET_BUILTIN(__nvvm_atom_cta_add_gen_l, "LiLiD*Li", "n", SM_60)
503TARGET_BUILTIN(__nvvm_atom_sys_add_gen_l, "LiLiD*Li", "n", SM_60)
504BUILTIN(__nvvm_atom_add_gen_ll, "LLiLLiD*LLi", "n")
505TARGET_BUILTIN(__nvvm_atom_cta_add_gen_ll, "LLiLLiD*LLi", "n", SM_60)
506TARGET_BUILTIN(__nvvm_atom_sys_add_gen_ll, "LLiLLiD*LLi", "n", SM_60)
507BUILTIN(__nvvm_atom_add_gen_f, "ffD*f", "n")
508TARGET_BUILTIN(__nvvm_atom_cta_add_gen_f, "ffD*f", "n", SM_60)
509TARGET_BUILTIN(__nvvm_atom_sys_add_gen_f, "ffD*f", "n", SM_60)
510TARGET_BUILTIN(__nvvm_atom_add_gen_d, "ddD*d", "n", SM_60)
511TARGET_BUILTIN(__nvvm_atom_cta_add_gen_d, "ddD*d", "n", SM_60)
512TARGET_BUILTIN(__nvvm_atom_sys_add_gen_d, "ddD*d", "n", SM_60)
513
514BUILTIN(__nvvm_atom_sub_gen_i, "iiD*i", "n")
515BUILTIN(__nvvm_atom_sub_gen_l, "LiLiD*Li", "n")
516BUILTIN(__nvvm_atom_sub_gen_ll, "LLiLLiD*LLi", "n")
517
518BUILTIN(__nvvm_atom_xchg_gen_i, "iiD*i", "n")
519TARGET_BUILTIN(__nvvm_atom_cta_xchg_gen_i, "iiD*i", "n", SM_60)
520TARGET_BUILTIN(__nvvm_atom_sys_xchg_gen_i, "iiD*i", "n", SM_60)
521BUILTIN(__nvvm_atom_xchg_gen_l, "LiLiD*Li", "n")
522TARGET_BUILTIN(__nvvm_atom_cta_xchg_gen_l, "LiLiD*Li", "n", SM_60)
523TARGET_BUILTIN(__nvvm_atom_sys_xchg_gen_l, "LiLiD*Li", "n", SM_60)
524BUILTIN(__nvvm_atom_xchg_gen_ll, "LLiLLiD*LLi", "n")
525TARGET_BUILTIN(__nvvm_atom_cta_xchg_gen_ll, "LLiLLiD*LLi", "n", SM_60)
526TARGET_BUILTIN(__nvvm_atom_sys_xchg_gen_ll, "LLiLLiD*LLi", "n", SM_60)
527
528BUILTIN(__nvvm_atom_max_gen_i, "iiD*i", "n")
529TARGET_BUILTIN(__nvvm_atom_cta_max_gen_i, "iiD*i", "n", SM_60)
530TARGET_BUILTIN(__nvvm_atom_sys_max_gen_i, "iiD*i", "n", SM_60)
531BUILTIN(__nvvm_atom_max_gen_ui, "UiUiD*Ui", "n")
532TARGET_BUILTIN(__nvvm_atom_cta_max_gen_ui, "UiUiD*Ui", "n", SM_60)
533TARGET_BUILTIN(__nvvm_atom_sys_max_gen_ui, "UiUiD*Ui", "n", SM_60)
534BUILTIN(__nvvm_atom_max_gen_l, "LiLiD*Li", "n")
535TARGET_BUILTIN(__nvvm_atom_cta_max_gen_l, "LiLiD*Li", "n", SM_60)
536TARGET_BUILTIN(__nvvm_atom_sys_max_gen_l, "LiLiD*Li", "n", SM_60)
537BUILTIN(__nvvm_atom_max_gen_ul, "ULiULiD*ULi", "n")
538TARGET_BUILTIN(__nvvm_atom_cta_max_gen_ul, "ULiULiD*ULi", "n", SM_60)
539TARGET_BUILTIN(__nvvm_atom_sys_max_gen_ul, "ULiULiD*ULi", "n", SM_60)
540BUILTIN(__nvvm_atom_max_gen_ll, "LLiLLiD*LLi", "n")
541TARGET_BUILTIN(__nvvm_atom_cta_max_gen_ll, "LLiLLiD*LLi", "n", SM_60)
542TARGET_BUILTIN(__nvvm_atom_sys_max_gen_ll, "LLiLLiD*LLi", "n", SM_60)
543BUILTIN(__nvvm_atom_max_gen_ull, "ULLiULLiD*ULLi", "n")
544TARGET_BUILTIN(__nvvm_atom_cta_max_gen_ull, "ULLiULLiD*ULLi", "n", SM_60)
545TARGET_BUILTIN(__nvvm_atom_sys_max_gen_ull, "ULLiULLiD*ULLi", "n", SM_60)
546
547BUILTIN(__nvvm_atom_min_gen_i, "iiD*i", "n")
548TARGET_BUILTIN(__nvvm_atom_cta_min_gen_i, "iiD*i", "n", SM_60)
549TARGET_BUILTIN(__nvvm_atom_sys_min_gen_i, "iiD*i", "n", SM_60)
550BUILTIN(__nvvm_atom_min_gen_ui, "UiUiD*Ui", "n")
551TARGET_BUILTIN(__nvvm_atom_cta_min_gen_ui, "UiUiD*Ui", "n", SM_60)
552TARGET_BUILTIN(__nvvm_atom_sys_min_gen_ui, "UiUiD*Ui", "n", SM_60)
553BUILTIN(__nvvm_atom_min_gen_l, "LiLiD*Li", "n")
554TARGET_BUILTIN(__nvvm_atom_cta_min_gen_l, "LiLiD*Li", "n", SM_60)
555TARGET_BUILTIN(__nvvm_atom_sys_min_gen_l, "LiLiD*Li", "n", SM_60)
556BUILTIN(__nvvm_atom_min_gen_ul, "ULiULiD*ULi", "n")
557TARGET_BUILTIN(__nvvm_atom_cta_min_gen_ul, "ULiULiD*ULi", "n", SM_60)
558TARGET_BUILTIN(__nvvm_atom_sys_min_gen_ul, "ULiULiD*ULi", "n", SM_60)
559BUILTIN(__nvvm_atom_min_gen_ll, "LLiLLiD*LLi", "n")
560TARGET_BUILTIN(__nvvm_atom_cta_min_gen_ll, "LLiLLiD*LLi", "n", SM_60)
561TARGET_BUILTIN(__nvvm_atom_sys_min_gen_ll, "LLiLLiD*LLi", "n", SM_60)
562BUILTIN(__nvvm_atom_min_gen_ull, "ULLiULLiD*ULLi", "n")
563TARGET_BUILTIN(__nvvm_atom_cta_min_gen_ull, "ULLiULLiD*ULLi", "n", SM_60)
564TARGET_BUILTIN(__nvvm_atom_sys_min_gen_ull, "ULLiULLiD*ULLi", "n", SM_60)
565
566BUILTIN(__nvvm_atom_inc_gen_ui, "UiUiD*Ui", "n")
567TARGET_BUILTIN(__nvvm_atom_cta_inc_gen_ui, "UiUiD*Ui", "n", SM_60)
568TARGET_BUILTIN(__nvvm_atom_sys_inc_gen_ui, "UiUiD*Ui", "n", SM_60)
569BUILTIN(__nvvm_atom_dec_gen_ui, "UiUiD*Ui", "n")
570TARGET_BUILTIN(__nvvm_atom_cta_dec_gen_ui, "UiUiD*Ui", "n", SM_60)
571TARGET_BUILTIN(__nvvm_atom_sys_dec_gen_ui, "UiUiD*Ui", "n", SM_60)
572
573BUILTIN(__nvvm_atom_and_gen_i, "iiD*i", "n")
574TARGET_BUILTIN(__nvvm_atom_cta_and_gen_i, "iiD*i", "n", SM_60)
575TARGET_BUILTIN(__nvvm_atom_sys_and_gen_i, "iiD*i", "n", SM_60)
576BUILTIN(__nvvm_atom_and_gen_l, "LiLiD*Li", "n")
577TARGET_BUILTIN(__nvvm_atom_cta_and_gen_l, "LiLiD*Li", "n", SM_60)
578TARGET_BUILTIN(__nvvm_atom_sys_and_gen_l, "LiLiD*Li", "n", SM_60)
579BUILTIN(__nvvm_atom_and_gen_ll, "LLiLLiD*LLi", "n")
580TARGET_BUILTIN(__nvvm_atom_cta_and_gen_ll, "LLiLLiD*LLi", "n", SM_60)
581TARGET_BUILTIN(__nvvm_atom_sys_and_gen_ll, "LLiLLiD*LLi", "n", SM_60)
582
583BUILTIN(__nvvm_atom_or_gen_i, "iiD*i", "n")
584TARGET_BUILTIN(__nvvm_atom_cta_or_gen_i, "iiD*i", "n", SM_60)
585TARGET_BUILTIN(__nvvm_atom_sys_or_gen_i, "iiD*i", "n", SM_60)
586BUILTIN(__nvvm_atom_or_gen_l, "LiLiD*Li", "n")
587TARGET_BUILTIN(__nvvm_atom_cta_or_gen_l, "LiLiD*Li", "n", SM_60)
588TARGET_BUILTIN(__nvvm_atom_sys_or_gen_l, "LiLiD*Li", "n", SM_60)
589BUILTIN(__nvvm_atom_or_gen_ll, "LLiLLiD*LLi", "n")
590TARGET_BUILTIN(__nvvm_atom_cta_or_gen_ll, "LLiLLiD*LLi", "n", SM_60)
591TARGET_BUILTIN(__nvvm_atom_sys_or_gen_ll, "LLiLLiD*LLi", "n", SM_60)
592
593BUILTIN(__nvvm_atom_xor_gen_i, "iiD*i", "n")
594TARGET_BUILTIN(__nvvm_atom_cta_xor_gen_i, "iiD*i", "n", SM_60)
595TARGET_BUILTIN(__nvvm_atom_sys_xor_gen_i, "iiD*i", "n", SM_60)
596BUILTIN(__nvvm_atom_xor_gen_l, "LiLiD*Li", "n")
597TARGET_BUILTIN(__nvvm_atom_cta_xor_gen_l, "LiLiD*Li", "n", SM_60)
598TARGET_BUILTIN(__nvvm_atom_sys_xor_gen_l, "LiLiD*Li", "n", SM_60)
599BUILTIN(__nvvm_atom_xor_gen_ll, "LLiLLiD*LLi", "n")
600TARGET_BUILTIN(__nvvm_atom_cta_xor_gen_ll, "LLiLLiD*LLi", "n", SM_60)
601TARGET_BUILTIN(__nvvm_atom_sys_xor_gen_ll, "LLiLLiD*LLi", "n", SM_60)
602
603BUILTIN(__nvvm_atom_cas_gen_i, "iiD*ii", "n")
604TARGET_BUILTIN(__nvvm_atom_cta_cas_gen_i, "iiD*ii", "n", SM_60)
605TARGET_BUILTIN(__nvvm_atom_sys_cas_gen_i, "iiD*ii", "n", SM_60)
606BUILTIN(__nvvm_atom_cas_gen_l, "LiLiD*LiLi", "n")
607TARGET_BUILTIN(__nvvm_atom_cta_cas_gen_l, "LiLiD*LiLi", "n", SM_60)
608TARGET_BUILTIN(__nvvm_atom_sys_cas_gen_l, "LiLiD*LiLi", "n", SM_60)
609BUILTIN(__nvvm_atom_cas_gen_ll, "LLiLLiD*LLiLLi", "n")
610TARGET_BUILTIN(__nvvm_atom_cta_cas_gen_ll, "LLiLLiD*LLiLLi", "n", SM_60)
611TARGET_BUILTIN(__nvvm_atom_sys_cas_gen_ll, "LLiLLiD*LLiLLi", "n", SM_60)
612
613// Compiler Error Warn
614BUILTIN(__nvvm_compiler_error, "vcC*4", "n")
615BUILTIN(__nvvm_compiler_warn, "vcC*4", "n")
616
617// __ldg. This is not implemented as a builtin by nvcc.
618BUILTIN(__nvvm_ldg_c, "ccC*", "")
619BUILTIN(__nvvm_ldg_s, "ssC*", "")
620BUILTIN(__nvvm_ldg_i, "iiC*", "")
621BUILTIN(__nvvm_ldg_l, "LiLiC*", "")
622BUILTIN(__nvvm_ldg_ll, "LLiLLiC*", "")
623
624BUILTIN(__nvvm_ldg_uc, "UcUcC*", "")
625BUILTIN(__nvvm_ldg_us, "UsUsC*", "")
626BUILTIN(__nvvm_ldg_ui, "UiUiC*", "")
627BUILTIN(__nvvm_ldg_ul, "ULiULiC*", "")
628BUILTIN(__nvvm_ldg_ull, "ULLiULLiC*", "")
629
630BUILTIN(__nvvm_ldg_f, "ffC*", "")
631BUILTIN(__nvvm_ldg_d, "ddC*", "")
632
633BUILTIN(__nvvm_ldg_c2, "E2cE2cC*", "")
634BUILTIN(__nvvm_ldg_c4, "E4cE4cC*", "")
635BUILTIN(__nvvm_ldg_s2, "E2sE2sC*", "")
636BUILTIN(__nvvm_ldg_s4, "E4sE4sC*", "")
637BUILTIN(__nvvm_ldg_i2, "E2iE2iC*", "")
638BUILTIN(__nvvm_ldg_i4, "E4iE4iC*", "")
639BUILTIN(__nvvm_ldg_ll2, "E2LLiE2LLiC*", "")
640
641BUILTIN(__nvvm_ldg_uc2, "E2UcE2UcC*", "")
642BUILTIN(__nvvm_ldg_uc4, "E4UcE4UcC*", "")
643BUILTIN(__nvvm_ldg_us2, "E2UsE2UsC*", "")
644BUILTIN(__nvvm_ldg_us4, "E4UsE4UsC*", "")
645BUILTIN(__nvvm_ldg_ui2, "E2UiE2UiC*", "")
646BUILTIN(__nvvm_ldg_ui4, "E4UiE4UiC*", "")
647BUILTIN(__nvvm_ldg_ull2, "E2ULLiE2ULLiC*", "")
648
649BUILTIN(__nvvm_ldg_f2, "E2fE2fC*", "")
650BUILTIN(__nvvm_ldg_f4, "E4fE4fC*", "")
651BUILTIN(__nvvm_ldg_d2, "E2dE2dC*", "")
652
653// Builtins to support WMMA instructions on sm_70
654TARGET_BUILTIN(__hmma_m16n16k16_ld_a, "vi*iC*UiIi", "", AND(SM_70,PTX60))
655TARGET_BUILTIN(__hmma_m16n16k16_ld_b, "vi*iC*UiIi", "", AND(SM_70,PTX60))
656TARGET_BUILTIN(__hmma_m16n16k16_ld_c_f16, "vi*iC*UiIi", "", AND(SM_70,PTX60))
657TARGET_BUILTIN(__hmma_m16n16k16_ld_c_f32, "vf*fC*UiIi", "", AND(SM_70,PTX60))
658TARGET_BUILTIN(__hmma_m16n16k16_st_c_f16, "vi*i*UiIi", "", AND(SM_70,PTX60))
659TARGET_BUILTIN(__hmma_m16n16k16_st_c_f32, "vf*f*UiIi", "", AND(SM_70,PTX60))
660
661TARGET_BUILTIN(__hmma_m32n8k16_ld_a, "vi*iC*UiIi", "", AND(SM_70,PTX61))
662TARGET_BUILTIN(__hmma_m32n8k16_ld_b, "vi*iC*UiIi", "", AND(SM_70,PTX61))
663TARGET_BUILTIN(__hmma_m32n8k16_ld_c_f16, "vi*iC*UiIi", "", AND(SM_70,PTX61))
664TARGET_BUILTIN(__hmma_m32n8k16_ld_c_f32, "vf*fC*UiIi", "", AND(SM_70,PTX61))
665TARGET_BUILTIN(__hmma_m32n8k16_st_c_f16, "vi*i*UiIi", "", AND(SM_70,PTX61))
666TARGET_BUILTIN(__hmma_m32n8k16_st_c_f32, "vf*f*UiIi", "", AND(SM_70,PTX61))
667
668TARGET_BUILTIN(__hmma_m8n32k16_ld_a, "vi*iC*UiIi", "", AND(SM_70,PTX61))
669TARGET_BUILTIN(__hmma_m8n32k16_ld_b, "vi*iC*UiIi", "", AND(SM_70,PTX61))
670TARGET_BUILTIN(__hmma_m8n32k16_ld_c_f16, "vi*iC*UiIi", "", AND(SM_70,PTX61))
671TARGET_BUILTIN(__hmma_m8n32k16_ld_c_f32, "vf*fC*UiIi", "", AND(SM_70,PTX61))
672TARGET_BUILTIN(__hmma_m8n32k16_st_c_f16, "vi*i*UiIi", "", AND(SM_70,PTX61))
673TARGET_BUILTIN(__hmma_m8n32k16_st_c_f32, "vf*f*UiIi", "", AND(SM_70,PTX61))
674
675TARGET_BUILTIN(__hmma_m16n16k16_mma_f16f16, "vi*iC*iC*iC*IiIi", "", AND(SM_70,PTX60))
676TARGET_BUILTIN(__hmma_m16n16k16_mma_f32f16, "vf*iC*iC*iC*IiIi", "", AND(SM_70,PTX60))
677TARGET_BUILTIN(__hmma_m16n16k16_mma_f32f32, "vf*iC*iC*fC*IiIi", "", AND(SM_70,PTX60))
678TARGET_BUILTIN(__hmma_m16n16k16_mma_f16f32, "vi*iC*iC*fC*IiIi", "", AND(SM_70,PTX60))
679
680TARGET_BUILTIN(__hmma_m32n8k16_mma_f16f16, "vi*iC*iC*iC*IiIi", "", AND(SM_70,PTX61))
681TARGET_BUILTIN(__hmma_m32n8k16_mma_f32f16, "vf*iC*iC*iC*IiIi", "", AND(SM_70,PTX61))
682TARGET_BUILTIN(__hmma_m32n8k16_mma_f32f32, "vf*iC*iC*fC*IiIi", "", AND(SM_70,PTX61))
683TARGET_BUILTIN(__hmma_m32n8k16_mma_f16f32, "vi*iC*iC*fC*IiIi", "", AND(SM_70,PTX61))
684
685TARGET_BUILTIN(__hmma_m8n32k16_mma_f16f16, "vi*iC*iC*iC*IiIi", "", AND(SM_70,PTX61))
686TARGET_BUILTIN(__hmma_m8n32k16_mma_f32f16, "vf*iC*iC*iC*IiIi", "", AND(SM_70,PTX61))
687TARGET_BUILTIN(__hmma_m8n32k16_mma_f32f32, "vf*iC*iC*fC*IiIi", "", AND(SM_70,PTX61))
688TARGET_BUILTIN(__hmma_m8n32k16_mma_f16f32, "vi*iC*iC*fC*IiIi", "", AND(SM_70,PTX61))
689
690// Builtins to support integer and sub-integer WMMA instructions on sm_72/sm_75
691TARGET_BUILTIN(__bmma_m8n8k128_ld_a_b1, "vi*iC*UiIi", "", AND(SM_75,PTX63))
692TARGET_BUILTIN(__bmma_m8n8k128_ld_b_b1, "vi*iC*UiIi", "", AND(SM_75,PTX63))
693TARGET_BUILTIN(__bmma_m8n8k128_ld_c, "vi*iC*UiIi", "", AND(SM_75,PTX63))
694TARGET_BUILTIN(__bmma_m8n8k128_mma_xor_popc_b1, "vi*iC*iC*iC*Ii", "", AND(SM_75,PTX63))
695TARGET_BUILTIN(__bmma_m8n8k128_st_c_i32, "vi*iC*UiIi", "", AND(SM_75,PTX63))
696TARGET_BUILTIN(__imma_m16n16k16_ld_a_s8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
697TARGET_BUILTIN(__imma_m16n16k16_ld_a_u8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
698TARGET_BUILTIN(__imma_m16n16k16_ld_b_s8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
699TARGET_BUILTIN(__imma_m16n16k16_ld_b_u8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
700TARGET_BUILTIN(__imma_m16n16k16_ld_c, "vi*iC*UiIi", "", AND(SM_72,PTX63))
701TARGET_BUILTIN(__imma_m16n16k16_mma_s8, "vi*iC*iC*iC*IiIi", "", AND(SM_72,PTX63))
702TARGET_BUILTIN(__imma_m16n16k16_mma_u8, "vi*iC*iC*iC*IiIi", "", AND(SM_72,PTX63))
703TARGET_BUILTIN(__imma_m16n16k16_st_c_i32, "vi*iC*UiIi", "", AND(SM_72,PTX63))
704TARGET_BUILTIN(__imma_m32n8k16_ld_a_s8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
705TARGET_BUILTIN(__imma_m32n8k16_ld_a_u8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
706TARGET_BUILTIN(__imma_m32n8k16_ld_b_s8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
707TARGET_BUILTIN(__imma_m32n8k16_ld_b_u8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
708TARGET_BUILTIN(__imma_m32n8k16_ld_c, "vi*iC*UiIi", "", AND(SM_72,PTX63))
709TARGET_BUILTIN(__imma_m32n8k16_mma_s8, "vi*iC*iC*iC*IiIi", "", AND(SM_72,PTX63))
710TARGET_BUILTIN(__imma_m32n8k16_mma_u8, "vi*iC*iC*iC*IiIi", "", AND(SM_72,PTX63))
711TARGET_BUILTIN(__imma_m32n8k16_st_c_i32, "vi*iC*UiIi", "", AND(SM_72,PTX63))
712TARGET_BUILTIN(__imma_m8n32k16_ld_a_s8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
713TARGET_BUILTIN(__imma_m8n32k16_ld_a_u8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
714TARGET_BUILTIN(__imma_m8n32k16_ld_b_s8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
715TARGET_BUILTIN(__imma_m8n32k16_ld_b_u8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
716TARGET_BUILTIN(__imma_m8n32k16_ld_c, "vi*iC*UiIi", "", AND(SM_72,PTX63))
717TARGET_BUILTIN(__imma_m8n32k16_mma_s8, "vi*iC*iC*iC*IiIi", "", AND(SM_72,PTX63))
718TARGET_BUILTIN(__imma_m8n32k16_mma_u8, "vi*iC*iC*iC*IiIi", "", AND(SM_72,PTX63))
719TARGET_BUILTIN(__imma_m8n32k16_st_c_i32, "vi*iC*UiIi", "", AND(SM_72,PTX63))
720TARGET_BUILTIN(__imma_m8n8k32_ld_a_s4, "vi*iC*UiIi", "", AND(SM_75,PTX63))
721TARGET_BUILTIN(__imma_m8n8k32_ld_a_u4, "vi*iC*UiIi", "", AND(SM_75,PTX63))
722TARGET_BUILTIN(__imma_m8n8k32_ld_b_s4, "vi*iC*UiIi", "", AND(SM_75,PTX63))
723TARGET_BUILTIN(__imma_m8n8k32_ld_b_u4, "vi*iC*UiIi", "", AND(SM_75,PTX63))
724TARGET_BUILTIN(__imma_m8n8k32_ld_c, "vi*iC*UiIi", "", AND(SM_75,PTX63))
725TARGET_BUILTIN(__imma_m8n8k32_mma_s4, "vi*iC*iC*iC*IiIi", "", AND(SM_75,PTX63))
726TARGET_BUILTIN(__imma_m8n8k32_mma_u4, "vi*iC*iC*iC*IiIi", "", AND(SM_75,PTX63))
727TARGET_BUILTIN(__imma_m8n8k32_st_c_i32, "vi*iC*UiIi", "", AND(SM_75,PTX63))
728
729#undef BUILTIN
730#undef TARGET_BUILTIN
731#pragma pop_macro("AND")
732#pragma pop_macro("SM_60")
733#pragma pop_macro("SM_70")
734#pragma pop_macro("SM_72")
735#pragma pop_macro("SM_75")
736#pragma pop_macro("SM_80")
737#pragma pop_macro("SM_86")
738#pragma pop_macro("PTX60")
739#pragma pop_macro("PTX61")
740#pragma pop_macro("PTX63")
741#pragma pop_macro("PTX64")
742#pragma pop_macro("PTX65")
743#pragma pop_macro("PTX70")
744#pragma pop_macro("PTX71")
745#pragma pop_macro("PTX72")
746