1 | /* Enumerate available IFUNC implementations of a function. x86-64 version. |
2 | Copyright (C) 2012-2022 Free Software Foundation, Inc. |
3 | This file is part of the GNU C Library. |
4 | |
5 | The GNU C Library is free software; you can redistribute it and/or |
6 | modify it under the terms of the GNU Lesser General Public |
7 | License as published by the Free Software Foundation; either |
8 | version 2.1 of the License, or (at your option) any later version. |
9 | |
10 | The GNU C Library is distributed in the hope that it will be useful, |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
13 | Lesser General Public License for more details. |
14 | |
15 | You should have received a copy of the GNU Lesser General Public |
16 | License along with the GNU C Library; if not, see |
17 | <https://www.gnu.org/licenses/>. */ |
18 | |
19 | #include <assert.h> |
20 | #include <string.h> |
21 | #include <wchar.h> |
22 | #include <ifunc-impl-list.h> |
23 | #include <sysdep.h> |
24 | #include "init-arch.h" |
25 | |
26 | /* Maximum number of IFUNC implementations. */ |
27 | #define MAX_IFUNC 5 |
28 | |
29 | /* Fill ARRAY of MAX elements with IFUNC implementations for function |
30 | NAME supported on target machine and return the number of valid |
31 | entries. */ |
32 | |
33 | size_t |
34 | __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, |
35 | size_t max) |
36 | { |
37 | assert (max >= MAX_IFUNC); |
38 | |
39 | size_t i = 0; |
40 | |
41 | /* Support sysdeps/x86_64/multiarch/memcmpeq.c. */ |
42 | IFUNC_IMPL (i, name, __memcmpeq, |
43 | IFUNC_IMPL_ADD (array, i, __memcmpeq, |
44 | (CPU_FEATURE_USABLE (AVX2) |
45 | && CPU_FEATURE_USABLE (BMI2)), |
46 | __memcmpeq_avx2) |
47 | IFUNC_IMPL_ADD (array, i, __memcmpeq, |
48 | (CPU_FEATURE_USABLE (AVX2) |
49 | && CPU_FEATURE_USABLE (BMI2) |
50 | && CPU_FEATURE_USABLE (RTM)), |
51 | __memcmpeq_avx2_rtm) |
52 | IFUNC_IMPL_ADD (array, i, __memcmpeq, |
53 | (CPU_FEATURE_USABLE (AVX512VL) |
54 | && CPU_FEATURE_USABLE (AVX512BW) |
55 | && CPU_FEATURE_USABLE (BMI2)), |
56 | __memcmpeq_evex) |
57 | IFUNC_IMPL_ADD (array, i, __memcmpeq, 1, __memcmpeq_sse2)) |
58 | |
59 | /* Support sysdeps/x86_64/multiarch/memchr.c. */ |
60 | IFUNC_IMPL (i, name, memchr, |
61 | IFUNC_IMPL_ADD (array, i, memchr, |
62 | (CPU_FEATURE_USABLE (AVX2) |
63 | && CPU_FEATURE_USABLE (BMI2)), |
64 | __memchr_avx2) |
65 | IFUNC_IMPL_ADD (array, i, memchr, |
66 | (CPU_FEATURE_USABLE (AVX2) |
67 | && CPU_FEATURE_USABLE (BMI2) |
68 | && CPU_FEATURE_USABLE (RTM)), |
69 | __memchr_avx2_rtm) |
70 | IFUNC_IMPL_ADD (array, i, memchr, |
71 | (CPU_FEATURE_USABLE (AVX512VL) |
72 | && CPU_FEATURE_USABLE (AVX512BW) |
73 | && CPU_FEATURE_USABLE (BMI2)), |
74 | __memchr_evex) |
75 | IFUNC_IMPL_ADD (array, i, memchr, |
76 | (CPU_FEATURE_USABLE (AVX512VL) |
77 | && CPU_FEATURE_USABLE (AVX512BW) |
78 | && CPU_FEATURE_USABLE (BMI2)), |
79 | __memchr_evex_rtm) |
80 | IFUNC_IMPL_ADD (array, i, memchr, 1, __memchr_sse2)) |
81 | |
82 | /* Support sysdeps/x86_64/multiarch/memcmp.c. */ |
83 | IFUNC_IMPL (i, name, memcmp, |
84 | IFUNC_IMPL_ADD (array, i, memcmp, |
85 | (CPU_FEATURE_USABLE (AVX2) |
86 | && CPU_FEATURE_USABLE (BMI2) |
87 | && CPU_FEATURE_USABLE (MOVBE)), |
88 | __memcmp_avx2_movbe) |
89 | IFUNC_IMPL_ADD (array, i, memcmp, |
90 | (CPU_FEATURE_USABLE (AVX2) |
91 | && CPU_FEATURE_USABLE (BMI2) |
92 | && CPU_FEATURE_USABLE (MOVBE) |
93 | && CPU_FEATURE_USABLE (RTM)), |
94 | __memcmp_avx2_movbe_rtm) |
95 | IFUNC_IMPL_ADD (array, i, memcmp, |
96 | (CPU_FEATURE_USABLE (AVX512VL) |
97 | && CPU_FEATURE_USABLE (AVX512BW) |
98 | && CPU_FEATURE_USABLE (BMI2) |
99 | && CPU_FEATURE_USABLE (MOVBE)), |
100 | __memcmp_evex_movbe) |
101 | IFUNC_IMPL_ADD (array, i, memcmp, CPU_FEATURE_USABLE (SSSE3), |
102 | __memcmp_ssse3) |
103 | IFUNC_IMPL_ADD (array, i, memcmp, 1, __memcmp_sse2)) |
104 | |
105 | #ifdef SHARED |
106 | /* Support sysdeps/x86_64/multiarch/memmove_chk.c. */ |
107 | IFUNC_IMPL (i, name, __memmove_chk, |
108 | IFUNC_IMPL_ADD (array, i, __memmove_chk, |
109 | CPU_FEATURE_USABLE (AVX512F), |
110 | __memmove_chk_avx512_no_vzeroupper) |
111 | IFUNC_IMPL_ADD (array, i, __memmove_chk, |
112 | CPU_FEATURE_USABLE (AVX512VL), |
113 | __memmove_chk_avx512_unaligned) |
114 | IFUNC_IMPL_ADD (array, i, __memmove_chk, |
115 | CPU_FEATURE_USABLE (AVX512VL), |
116 | __memmove_chk_avx512_unaligned_erms) |
117 | IFUNC_IMPL_ADD (array, i, __memmove_chk, |
118 | CPU_FEATURE_USABLE (AVX), |
119 | __memmove_chk_avx_unaligned) |
120 | IFUNC_IMPL_ADD (array, i, __memmove_chk, |
121 | CPU_FEATURE_USABLE (AVX), |
122 | __memmove_chk_avx_unaligned_erms) |
123 | IFUNC_IMPL_ADD (array, i, __memmove_chk, |
124 | (CPU_FEATURE_USABLE (AVX) |
125 | && CPU_FEATURE_USABLE (RTM)), |
126 | __memmove_chk_avx_unaligned_rtm) |
127 | IFUNC_IMPL_ADD (array, i, __memmove_chk, |
128 | (CPU_FEATURE_USABLE (AVX) |
129 | && CPU_FEATURE_USABLE (RTM)), |
130 | __memmove_chk_avx_unaligned_erms_rtm) |
131 | IFUNC_IMPL_ADD (array, i, __memmove_chk, |
132 | CPU_FEATURE_USABLE (AVX512VL), |
133 | __memmove_chk_evex_unaligned) |
134 | IFUNC_IMPL_ADD (array, i, __memmove_chk, |
135 | CPU_FEATURE_USABLE (AVX512VL), |
136 | __memmove_chk_evex_unaligned_erms) |
137 | IFUNC_IMPL_ADD (array, i, __memmove_chk, |
138 | CPU_FEATURE_USABLE (SSSE3), |
139 | __memmove_chk_ssse3_back) |
140 | IFUNC_IMPL_ADD (array, i, __memmove_chk, |
141 | CPU_FEATURE_USABLE (SSSE3), |
142 | __memmove_chk_ssse3) |
143 | IFUNC_IMPL_ADD (array, i, __memmove_chk, 1, |
144 | __memmove_chk_sse2_unaligned) |
145 | IFUNC_IMPL_ADD (array, i, __memmove_chk, 1, |
146 | __memmove_chk_sse2_unaligned_erms) |
147 | IFUNC_IMPL_ADD (array, i, __memmove_chk, 1, |
148 | __memmove_chk_erms)) |
149 | #endif |
150 | |
151 | /* Support sysdeps/x86_64/multiarch/memmove.c. */ |
152 | IFUNC_IMPL (i, name, memmove, |
153 | IFUNC_IMPL_ADD (array, i, memmove, |
154 | CPU_FEATURE_USABLE (AVX), |
155 | __memmove_avx_unaligned) |
156 | IFUNC_IMPL_ADD (array, i, memmove, |
157 | CPU_FEATURE_USABLE (AVX), |
158 | __memmove_avx_unaligned_erms) |
159 | IFUNC_IMPL_ADD (array, i, memmove, |
160 | (CPU_FEATURE_USABLE (AVX) |
161 | && CPU_FEATURE_USABLE (RTM)), |
162 | __memmove_avx_unaligned_rtm) |
163 | IFUNC_IMPL_ADD (array, i, memmove, |
164 | (CPU_FEATURE_USABLE (AVX) |
165 | && CPU_FEATURE_USABLE (RTM)), |
166 | __memmove_avx_unaligned_erms_rtm) |
167 | IFUNC_IMPL_ADD (array, i, memmove, |
168 | CPU_FEATURE_USABLE (AVX512VL), |
169 | __memmove_evex_unaligned) |
170 | IFUNC_IMPL_ADD (array, i, memmove, |
171 | CPU_FEATURE_USABLE (AVX512VL), |
172 | __memmove_evex_unaligned_erms) |
173 | IFUNC_IMPL_ADD (array, i, memmove, |
174 | CPU_FEATURE_USABLE (AVX512F), |
175 | __memmove_avx512_no_vzeroupper) |
176 | IFUNC_IMPL_ADD (array, i, memmove, |
177 | CPU_FEATURE_USABLE (AVX512VL), |
178 | __memmove_avx512_unaligned) |
179 | IFUNC_IMPL_ADD (array, i, memmove, |
180 | CPU_FEATURE_USABLE (AVX512VL), |
181 | __memmove_avx512_unaligned_erms) |
182 | IFUNC_IMPL_ADD (array, i, memmove, CPU_FEATURE_USABLE (SSSE3), |
183 | __memmove_ssse3_back) |
184 | IFUNC_IMPL_ADD (array, i, memmove, CPU_FEATURE_USABLE (SSSE3), |
185 | __memmove_ssse3) |
186 | IFUNC_IMPL_ADD (array, i, memmove, 1, __memmove_erms) |
187 | IFUNC_IMPL_ADD (array, i, memmove, 1, |
188 | __memmove_sse2_unaligned) |
189 | IFUNC_IMPL_ADD (array, i, memmove, 1, |
190 | __memmove_sse2_unaligned_erms)) |
191 | |
192 | /* Support sysdeps/x86_64/multiarch/memrchr.c. */ |
193 | IFUNC_IMPL (i, name, memrchr, |
194 | IFUNC_IMPL_ADD (array, i, memrchr, |
195 | (CPU_FEATURE_USABLE (AVX2) |
196 | && CPU_FEATURE_USABLE (BMI2) |
197 | && CPU_FEATURE_USABLE (LZCNT)), |
198 | __memrchr_avx2) |
199 | IFUNC_IMPL_ADD (array, i, memrchr, |
200 | (CPU_FEATURE_USABLE (AVX2) |
201 | && CPU_FEATURE_USABLE (BMI2) |
202 | && CPU_FEATURE_USABLE (LZCNT) |
203 | && CPU_FEATURE_USABLE (RTM)), |
204 | __memrchr_avx2_rtm) |
205 | IFUNC_IMPL_ADD (array, i, memrchr, |
206 | (CPU_FEATURE_USABLE (AVX512VL) |
207 | && CPU_FEATURE_USABLE (AVX512BW) |
208 | && CPU_FEATURE_USABLE (BMI2) |
209 | && CPU_FEATURE_USABLE (LZCNT)), |
210 | __memrchr_evex) |
211 | |
212 | IFUNC_IMPL_ADD (array, i, memrchr, 1, __memrchr_sse2)) |
213 | |
214 | #ifdef SHARED |
215 | /* Support sysdeps/x86_64/multiarch/memset_chk.c. */ |
216 | IFUNC_IMPL (i, name, __memset_chk, |
217 | IFUNC_IMPL_ADD (array, i, __memset_chk, 1, |
218 | __memset_chk_erms) |
219 | IFUNC_IMPL_ADD (array, i, __memset_chk, 1, |
220 | __memset_chk_sse2_unaligned) |
221 | IFUNC_IMPL_ADD (array, i, __memset_chk, 1, |
222 | __memset_chk_sse2_unaligned_erms) |
223 | IFUNC_IMPL_ADD (array, i, __memset_chk, |
224 | CPU_FEATURE_USABLE (AVX2), |
225 | __memset_chk_avx2_unaligned) |
226 | IFUNC_IMPL_ADD (array, i, __memset_chk, |
227 | CPU_FEATURE_USABLE (AVX2), |
228 | __memset_chk_avx2_unaligned_erms) |
229 | IFUNC_IMPL_ADD (array, i, __memset_chk, |
230 | (CPU_FEATURE_USABLE (AVX2) |
231 | && CPU_FEATURE_USABLE (RTM)), |
232 | __memset_chk_avx2_unaligned_rtm) |
233 | IFUNC_IMPL_ADD (array, i, __memset_chk, |
234 | (CPU_FEATURE_USABLE (AVX2) |
235 | && CPU_FEATURE_USABLE (RTM)), |
236 | __memset_chk_avx2_unaligned_erms_rtm) |
237 | IFUNC_IMPL_ADD (array, i, __memset_chk, |
238 | (CPU_FEATURE_USABLE (AVX512VL) |
239 | && CPU_FEATURE_USABLE (AVX512BW) |
240 | && CPU_FEATURE_USABLE (BMI2)), |
241 | __memset_chk_evex_unaligned) |
242 | IFUNC_IMPL_ADD (array, i, __memset_chk, |
243 | (CPU_FEATURE_USABLE (AVX512VL) |
244 | && CPU_FEATURE_USABLE (AVX512BW) |
245 | && CPU_FEATURE_USABLE (BMI2)), |
246 | __memset_chk_evex_unaligned_erms) |
247 | IFUNC_IMPL_ADD (array, i, __memset_chk, |
248 | (CPU_FEATURE_USABLE (AVX512VL) |
249 | && CPU_FEATURE_USABLE (AVX512BW) |
250 | && CPU_FEATURE_USABLE (BMI2)), |
251 | __memset_chk_avx512_unaligned_erms) |
252 | IFUNC_IMPL_ADD (array, i, __memset_chk, |
253 | (CPU_FEATURE_USABLE (AVX512VL) |
254 | && CPU_FEATURE_USABLE (AVX512BW) |
255 | && CPU_FEATURE_USABLE (BMI2)), |
256 | __memset_chk_avx512_unaligned) |
257 | IFUNC_IMPL_ADD (array, i, __memset_chk, |
258 | CPU_FEATURE_USABLE (AVX512F), |
259 | __memset_chk_avx512_no_vzeroupper) |
260 | ) |
261 | #endif |
262 | |
263 | /* Support sysdeps/x86_64/multiarch/memset.c. */ |
264 | IFUNC_IMPL (i, name, memset, |
265 | IFUNC_IMPL_ADD (array, i, memset, 1, |
266 | __memset_sse2_unaligned) |
267 | IFUNC_IMPL_ADD (array, i, memset, 1, |
268 | __memset_sse2_unaligned_erms) |
269 | IFUNC_IMPL_ADD (array, i, memset, 1, __memset_erms) |
270 | IFUNC_IMPL_ADD (array, i, memset, |
271 | CPU_FEATURE_USABLE (AVX2), |
272 | __memset_avx2_unaligned) |
273 | IFUNC_IMPL_ADD (array, i, memset, |
274 | CPU_FEATURE_USABLE (AVX2), |
275 | __memset_avx2_unaligned_erms) |
276 | IFUNC_IMPL_ADD (array, i, memset, |
277 | (CPU_FEATURE_USABLE (AVX2) |
278 | && CPU_FEATURE_USABLE (RTM)), |
279 | __memset_avx2_unaligned_rtm) |
280 | IFUNC_IMPL_ADD (array, i, memset, |
281 | (CPU_FEATURE_USABLE (AVX2) |
282 | && CPU_FEATURE_USABLE (RTM)), |
283 | __memset_avx2_unaligned_erms_rtm) |
284 | IFUNC_IMPL_ADD (array, i, memset, |
285 | (CPU_FEATURE_USABLE (AVX512VL) |
286 | && CPU_FEATURE_USABLE (AVX512BW) |
287 | && CPU_FEATURE_USABLE (BMI2)), |
288 | __memset_evex_unaligned) |
289 | IFUNC_IMPL_ADD (array, i, memset, |
290 | (CPU_FEATURE_USABLE (AVX512VL) |
291 | && CPU_FEATURE_USABLE (AVX512BW) |
292 | && CPU_FEATURE_USABLE (BMI2)), |
293 | __memset_evex_unaligned_erms) |
294 | IFUNC_IMPL_ADD (array, i, memset, |
295 | (CPU_FEATURE_USABLE (AVX512VL) |
296 | && CPU_FEATURE_USABLE (AVX512BW) |
297 | && CPU_FEATURE_USABLE (BMI2)), |
298 | __memset_avx512_unaligned_erms) |
299 | IFUNC_IMPL_ADD (array, i, memset, |
300 | (CPU_FEATURE_USABLE (AVX512VL) |
301 | && CPU_FEATURE_USABLE (AVX512BW) |
302 | && CPU_FEATURE_USABLE (BMI2)), |
303 | __memset_avx512_unaligned) |
304 | IFUNC_IMPL_ADD (array, i, memset, |
305 | CPU_FEATURE_USABLE (AVX512F), |
306 | __memset_avx512_no_vzeroupper) |
307 | ) |
308 | |
309 | /* Support sysdeps/x86_64/multiarch/rawmemchr.c. */ |
310 | IFUNC_IMPL (i, name, rawmemchr, |
311 | IFUNC_IMPL_ADD (array, i, rawmemchr, |
312 | (CPU_FEATURE_USABLE (AVX2) |
313 | && CPU_FEATURE_USABLE (BMI2)), |
314 | __rawmemchr_avx2) |
315 | IFUNC_IMPL_ADD (array, i, rawmemchr, |
316 | (CPU_FEATURE_USABLE (AVX2) |
317 | && CPU_FEATURE_USABLE (BMI2) |
318 | && CPU_FEATURE_USABLE (RTM)), |
319 | __rawmemchr_avx2_rtm) |
320 | IFUNC_IMPL_ADD (array, i, rawmemchr, |
321 | (CPU_FEATURE_USABLE (AVX512VL) |
322 | && CPU_FEATURE_USABLE (AVX512BW) |
323 | && CPU_FEATURE_USABLE (BMI2)), |
324 | __rawmemchr_evex) |
325 | IFUNC_IMPL_ADD (array, i, rawmemchr, |
326 | (CPU_FEATURE_USABLE (AVX512VL) |
327 | && CPU_FEATURE_USABLE (AVX512BW) |
328 | && CPU_FEATURE_USABLE (BMI2)), |
329 | __rawmemchr_evex_rtm) |
330 | IFUNC_IMPL_ADD (array, i, rawmemchr, 1, __rawmemchr_sse2)) |
331 | |
332 | /* Support sysdeps/x86_64/multiarch/strlen.c. */ |
333 | IFUNC_IMPL (i, name, strlen, |
334 | IFUNC_IMPL_ADD (array, i, strlen, |
335 | (CPU_FEATURE_USABLE (AVX2) |
336 | && CPU_FEATURE_USABLE (BMI2)), |
337 | __strlen_avx2) |
338 | IFUNC_IMPL_ADD (array, i, strlen, |
339 | (CPU_FEATURE_USABLE (AVX2) |
340 | && CPU_FEATURE_USABLE (BMI2) |
341 | && CPU_FEATURE_USABLE (RTM)), |
342 | __strlen_avx2_rtm) |
343 | IFUNC_IMPL_ADD (array, i, strlen, |
344 | (CPU_FEATURE_USABLE (AVX512VL) |
345 | && CPU_FEATURE_USABLE (AVX512BW) |
346 | && CPU_FEATURE_USABLE (BMI2)), |
347 | __strlen_evex) |
348 | IFUNC_IMPL_ADD (array, i, strlen, |
349 | (CPU_FEATURE_USABLE (AVX512VL) |
350 | && CPU_FEATURE_USABLE (AVX512BW) |
351 | && CPU_FEATURE_USABLE (BMI2)), |
352 | __strlen_evex512) |
353 | IFUNC_IMPL_ADD (array, i, strlen, 1, __strlen_sse2)) |
354 | |
355 | /* Support sysdeps/x86_64/multiarch/strnlen.c. */ |
356 | IFUNC_IMPL (i, name, strnlen, |
357 | IFUNC_IMPL_ADD (array, i, strnlen, |
358 | (CPU_FEATURE_USABLE (AVX2) |
359 | && CPU_FEATURE_USABLE (BMI2)), |
360 | __strnlen_avx2) |
361 | IFUNC_IMPL_ADD (array, i, strnlen, |
362 | (CPU_FEATURE_USABLE (AVX2) |
363 | && CPU_FEATURE_USABLE (BMI2) |
364 | && CPU_FEATURE_USABLE (RTM)), |
365 | __strnlen_avx2_rtm) |
366 | IFUNC_IMPL_ADD (array, i, strnlen, |
367 | (CPU_FEATURE_USABLE (AVX512VL) |
368 | && CPU_FEATURE_USABLE (AVX512BW) |
369 | && CPU_FEATURE_USABLE (BMI2)), |
370 | __strnlen_evex) |
371 | IFUNC_IMPL_ADD (array, i, strnlen, |
372 | (CPU_FEATURE_USABLE (AVX512VL) |
373 | && CPU_FEATURE_USABLE (AVX512BW) |
374 | && CPU_FEATURE_USABLE (BMI2)), |
375 | __strnlen_evex512) |
376 | IFUNC_IMPL_ADD (array, i, strnlen, 1, __strnlen_sse2)) |
377 | |
378 | /* Support sysdeps/x86_64/multiarch/stpncpy.c. */ |
379 | IFUNC_IMPL (i, name, stpncpy, |
380 | IFUNC_IMPL_ADD (array, i, stpncpy, CPU_FEATURE_USABLE (SSSE3), |
381 | __stpncpy_ssse3) |
382 | IFUNC_IMPL_ADD (array, i, stpncpy, CPU_FEATURE_USABLE (AVX2), |
383 | __stpncpy_avx2) |
384 | IFUNC_IMPL_ADD (array, i, stpncpy, |
385 | (CPU_FEATURE_USABLE (AVX2) |
386 | && CPU_FEATURE_USABLE (RTM)), |
387 | __stpncpy_avx2_rtm) |
388 | IFUNC_IMPL_ADD (array, i, stpncpy, |
389 | (CPU_FEATURE_USABLE (AVX512VL) |
390 | && CPU_FEATURE_USABLE (AVX512BW)), |
391 | __stpncpy_evex) |
392 | IFUNC_IMPL_ADD (array, i, stpncpy, 1, |
393 | __stpncpy_sse2_unaligned) |
394 | IFUNC_IMPL_ADD (array, i, stpncpy, 1, __stpncpy_sse2)) |
395 | |
396 | /* Support sysdeps/x86_64/multiarch/stpcpy.c. */ |
397 | IFUNC_IMPL (i, name, stpcpy, |
398 | IFUNC_IMPL_ADD (array, i, stpcpy, CPU_FEATURE_USABLE (SSSE3), |
399 | __stpcpy_ssse3) |
400 | IFUNC_IMPL_ADD (array, i, stpcpy, CPU_FEATURE_USABLE (AVX2), |
401 | __stpcpy_avx2) |
402 | IFUNC_IMPL_ADD (array, i, stpcpy, |
403 | (CPU_FEATURE_USABLE (AVX2) |
404 | && CPU_FEATURE_USABLE (RTM)), |
405 | __stpcpy_avx2_rtm) |
406 | IFUNC_IMPL_ADD (array, i, stpcpy, |
407 | (CPU_FEATURE_USABLE (AVX512VL) |
408 | && CPU_FEATURE_USABLE (AVX512BW)), |
409 | __stpcpy_evex) |
410 | IFUNC_IMPL_ADD (array, i, stpcpy, 1, __stpcpy_sse2_unaligned) |
411 | IFUNC_IMPL_ADD (array, i, stpcpy, 1, __stpcpy_sse2)) |
412 | |
413 | /* Support sysdeps/x86_64/multiarch/strcasecmp_l.c. */ |
414 | IFUNC_IMPL (i, name, strcasecmp, |
415 | IFUNC_IMPL_ADD (array, i, strcasecmp, |
416 | (CPU_FEATURE_USABLE (AVX512VL) |
417 | && CPU_FEATURE_USABLE (AVX512BW) |
418 | && CPU_FEATURE_USABLE (BMI2)), |
419 | __strcasecmp_evex) |
420 | IFUNC_IMPL_ADD (array, i, strcasecmp, |
421 | (CPU_FEATURE_USABLE (AVX2) |
422 | && CPU_FEATURE_USABLE (BMI2)), |
423 | __strcasecmp_avx2) |
424 | IFUNC_IMPL_ADD (array, i, strcasecmp, |
425 | (CPU_FEATURE_USABLE (AVX2) |
426 | && CPU_FEATURE_USABLE (BMI2) |
427 | && CPU_FEATURE_USABLE (RTM)), |
428 | __strcasecmp_avx2_rtm) |
429 | IFUNC_IMPL_ADD (array, i, strcasecmp, |
430 | CPU_FEATURE_USABLE (SSE4_2), |
431 | __strcasecmp_sse42) |
432 | IFUNC_IMPL_ADD (array, i, strcasecmp, |
433 | CPU_FEATURE_USABLE (SSSE3), |
434 | __strcasecmp_ssse3) |
435 | IFUNC_IMPL_ADD (array, i, strcasecmp, 1, __strcasecmp_sse2)) |
436 | |
437 | /* Support sysdeps/x86_64/multiarch/strcasecmp_l.c. */ |
438 | IFUNC_IMPL (i, name, strcasecmp_l, |
439 | IFUNC_IMPL_ADD (array, i, strcasecmp, |
440 | (CPU_FEATURE_USABLE (AVX512VL) |
441 | && CPU_FEATURE_USABLE (AVX512BW) |
442 | && CPU_FEATURE_USABLE (BMI2)), |
443 | __strcasecmp_l_evex) |
444 | IFUNC_IMPL_ADD (array, i, strcasecmp, |
445 | (CPU_FEATURE_USABLE (AVX2) |
446 | && CPU_FEATURE_USABLE (BMI2)), |
447 | __strcasecmp_l_avx2) |
448 | IFUNC_IMPL_ADD (array, i, strcasecmp, |
449 | (CPU_FEATURE_USABLE (AVX2) |
450 | && CPU_FEATURE_USABLE (BMI2) |
451 | && CPU_FEATURE_USABLE (RTM)), |
452 | __strcasecmp_l_avx2_rtm) |
453 | IFUNC_IMPL_ADD (array, i, strcasecmp_l, |
454 | CPU_FEATURE_USABLE (SSE4_2), |
455 | __strcasecmp_l_sse42) |
456 | IFUNC_IMPL_ADD (array, i, strcasecmp_l, |
457 | CPU_FEATURE_USABLE (SSSE3), |
458 | __strcasecmp_l_ssse3) |
459 | IFUNC_IMPL_ADD (array, i, strcasecmp_l, 1, |
460 | __strcasecmp_l_sse2)) |
461 | |
462 | /* Support sysdeps/x86_64/multiarch/strcat.c. */ |
463 | IFUNC_IMPL (i, name, strcat, |
464 | IFUNC_IMPL_ADD (array, i, strcat, CPU_FEATURE_USABLE (AVX2), |
465 | __strcat_avx2) |
466 | IFUNC_IMPL_ADD (array, i, strcat, |
467 | (CPU_FEATURE_USABLE (AVX2) |
468 | && CPU_FEATURE_USABLE (RTM)), |
469 | __strcat_avx2_rtm) |
470 | IFUNC_IMPL_ADD (array, i, strcat, |
471 | (CPU_FEATURE_USABLE (AVX512VL) |
472 | && CPU_FEATURE_USABLE (AVX512BW)), |
473 | __strcat_evex) |
474 | IFUNC_IMPL_ADD (array, i, strcat, CPU_FEATURE_USABLE (SSSE3), |
475 | __strcat_ssse3) |
476 | IFUNC_IMPL_ADD (array, i, strcat, 1, __strcat_sse2_unaligned) |
477 | IFUNC_IMPL_ADD (array, i, strcat, 1, __strcat_sse2)) |
478 | |
479 | /* Support sysdeps/x86_64/multiarch/strchr.c. */ |
480 | IFUNC_IMPL (i, name, strchr, |
481 | IFUNC_IMPL_ADD (array, i, strchr, |
482 | (CPU_FEATURE_USABLE (AVX2) |
483 | && CPU_FEATURE_USABLE (BMI2)), |
484 | __strchr_avx2) |
485 | IFUNC_IMPL_ADD (array, i, strchr, |
486 | (CPU_FEATURE_USABLE (AVX2) |
487 | && CPU_FEATURE_USABLE (BMI2) |
488 | && CPU_FEATURE_USABLE (RTM)), |
489 | __strchr_avx2_rtm) |
490 | IFUNC_IMPL_ADD (array, i, strchr, |
491 | (CPU_FEATURE_USABLE (AVX512VL) |
492 | && CPU_FEATURE_USABLE (AVX512BW) |
493 | && CPU_FEATURE_USABLE (BMI2)), |
494 | __strchr_evex) |
495 | IFUNC_IMPL_ADD (array, i, strchr, 1, __strchr_sse2_no_bsf) |
496 | IFUNC_IMPL_ADD (array, i, strchr, 1, __strchr_sse2)) |
497 | |
498 | /* Support sysdeps/x86_64/multiarch/strchrnul.c. */ |
499 | IFUNC_IMPL (i, name, strchrnul, |
500 | IFUNC_IMPL_ADD (array, i, strchrnul, |
501 | (CPU_FEATURE_USABLE (AVX2) |
502 | && CPU_FEATURE_USABLE (BMI2)), |
503 | __strchrnul_avx2) |
504 | IFUNC_IMPL_ADD (array, i, strchrnul, |
505 | (CPU_FEATURE_USABLE (AVX2) |
506 | && CPU_FEATURE_USABLE (BMI2) |
507 | && CPU_FEATURE_USABLE (RTM)), |
508 | __strchrnul_avx2_rtm) |
509 | IFUNC_IMPL_ADD (array, i, strchrnul, |
510 | (CPU_FEATURE_USABLE (AVX512VL) |
511 | && CPU_FEATURE_USABLE (AVX512BW) |
512 | && CPU_FEATURE_USABLE (BMI2)), |
513 | __strchrnul_evex) |
514 | IFUNC_IMPL_ADD (array, i, strchrnul, 1, __strchrnul_sse2)) |
515 | |
516 | /* Support sysdeps/x86_64/multiarch/strrchr.c. */ |
517 | IFUNC_IMPL (i, name, strrchr, |
518 | IFUNC_IMPL_ADD (array, i, strrchr, |
519 | (CPU_FEATURE_USABLE (AVX2) |
520 | && CPU_FEATURE_USABLE (BMI1) |
521 | && CPU_FEATURE_USABLE (BMI2)), |
522 | __strrchr_avx2) |
523 | IFUNC_IMPL_ADD (array, i, strrchr, |
524 | (CPU_FEATURE_USABLE (AVX2) |
525 | && CPU_FEATURE_USABLE (BMI1) |
526 | && CPU_FEATURE_USABLE (BMI2) |
527 | && CPU_FEATURE_USABLE (RTM)), |
528 | __strrchr_avx2_rtm) |
529 | IFUNC_IMPL_ADD (array, i, strrchr, |
530 | (CPU_FEATURE_USABLE (AVX512VL) |
531 | && CPU_FEATURE_USABLE (AVX512BW) |
532 | && CPU_FEATURE_USABLE (BMI1) |
533 | && CPU_FEATURE_USABLE (BMI2)), |
534 | __strrchr_evex) |
535 | IFUNC_IMPL_ADD (array, i, strrchr, 1, __strrchr_sse2)) |
536 | |
537 | /* Support sysdeps/x86_64/multiarch/strcmp.c. */ |
538 | IFUNC_IMPL (i, name, strcmp, |
539 | IFUNC_IMPL_ADD (array, i, strcmp, |
540 | (CPU_FEATURE_USABLE (AVX2) |
541 | && CPU_FEATURE_USABLE (BMI2)), |
542 | __strcmp_avx2) |
543 | IFUNC_IMPL_ADD (array, i, strcmp, |
544 | (CPU_FEATURE_USABLE (AVX2) |
545 | && CPU_FEATURE_USABLE (BMI2) |
546 | && CPU_FEATURE_USABLE (RTM)), |
547 | __strcmp_avx2_rtm) |
548 | IFUNC_IMPL_ADD (array, i, strcmp, |
549 | (CPU_FEATURE_USABLE (AVX512VL) |
550 | && CPU_FEATURE_USABLE (AVX512BW) |
551 | && CPU_FEATURE_USABLE (BMI2)), |
552 | __strcmp_evex) |
553 | IFUNC_IMPL_ADD (array, i, strcmp, CPU_FEATURE_USABLE (SSE4_2), |
554 | __strcmp_sse42) |
555 | IFUNC_IMPL_ADD (array, i, strcmp, CPU_FEATURE_USABLE (SSSE3), |
556 | __strcmp_ssse3) |
557 | IFUNC_IMPL_ADD (array, i, strcmp, 1, __strcmp_sse2_unaligned) |
558 | IFUNC_IMPL_ADD (array, i, strcmp, 1, __strcmp_sse2)) |
559 | |
560 | /* Support sysdeps/x86_64/multiarch/strcpy.c. */ |
561 | IFUNC_IMPL (i, name, strcpy, |
562 | IFUNC_IMPL_ADD (array, i, strcpy, CPU_FEATURE_USABLE (AVX2), |
563 | __strcpy_avx2) |
564 | IFUNC_IMPL_ADD (array, i, strcpy, |
565 | (CPU_FEATURE_USABLE (AVX2) |
566 | && CPU_FEATURE_USABLE (RTM)), |
567 | __strcpy_avx2_rtm) |
568 | IFUNC_IMPL_ADD (array, i, strcpy, |
569 | (CPU_FEATURE_USABLE (AVX512VL) |
570 | && CPU_FEATURE_USABLE (AVX512BW)), |
571 | __strcpy_evex) |
572 | IFUNC_IMPL_ADD (array, i, strcpy, CPU_FEATURE_USABLE (SSSE3), |
573 | __strcpy_ssse3) |
574 | IFUNC_IMPL_ADD (array, i, strcpy, 1, __strcpy_sse2_unaligned) |
575 | IFUNC_IMPL_ADD (array, i, strcpy, 1, __strcpy_sse2)) |
576 | |
577 | /* Support sysdeps/x86_64/multiarch/strcspn.c. */ |
578 | IFUNC_IMPL (i, name, strcspn, |
579 | IFUNC_IMPL_ADD (array, i, strcspn, CPU_FEATURE_USABLE (SSE4_2), |
580 | __strcspn_sse42) |
581 | IFUNC_IMPL_ADD (array, i, strcspn, 1, __strcspn_sse2)) |
582 | |
583 | /* Support sysdeps/x86_64/multiarch/strncase_l.c. */ |
584 | IFUNC_IMPL (i, name, strncasecmp, |
585 | IFUNC_IMPL_ADD (array, i, strncasecmp, |
586 | (CPU_FEATURE_USABLE (AVX512VL) |
587 | && CPU_FEATURE_USABLE (AVX512BW) |
588 | && CPU_FEATURE_USABLE (BMI2)), |
589 | __strncasecmp_evex) |
590 | IFUNC_IMPL_ADD (array, i, strncasecmp, |
591 | (CPU_FEATURE_USABLE (AVX2) |
592 | && CPU_FEATURE_USABLE (BMI2)), |
593 | __strncasecmp_avx2) |
594 | IFUNC_IMPL_ADD (array, i, strncasecmp, |
595 | (CPU_FEATURE_USABLE (AVX2) |
596 | && CPU_FEATURE_USABLE (BMI2) |
597 | && CPU_FEATURE_USABLE (RTM)), |
598 | __strncasecmp_avx2_rtm) |
599 | IFUNC_IMPL_ADD (array, i, strncasecmp, |
600 | CPU_FEATURE_USABLE (SSE4_2), |
601 | __strncasecmp_sse42) |
602 | IFUNC_IMPL_ADD (array, i, strncasecmp, |
603 | CPU_FEATURE_USABLE (SSSE3), |
604 | __strncasecmp_ssse3) |
605 | IFUNC_IMPL_ADD (array, i, strncasecmp, 1, |
606 | __strncasecmp_sse2)) |
607 | |
608 | /* Support sysdeps/x86_64/multiarch/strncase_l.c. */ |
609 | IFUNC_IMPL (i, name, strncasecmp_l, |
610 | IFUNC_IMPL_ADD (array, i, strncasecmp, |
611 | (CPU_FEATURE_USABLE (AVX512VL) |
612 | & CPU_FEATURE_USABLE (AVX512BW) |
613 | && CPU_FEATURE_USABLE (BMI2)), |
614 | __strncasecmp_l_evex) |
615 | IFUNC_IMPL_ADD (array, i, strncasecmp, |
616 | (CPU_FEATURE_USABLE (AVX2) |
617 | && CPU_FEATURE_USABLE (BMI2)), |
618 | __strncasecmp_l_avx2) |
619 | IFUNC_IMPL_ADD (array, i, strncasecmp, |
620 | (CPU_FEATURE_USABLE (AVX2) |
621 | && CPU_FEATURE_USABLE (BMI2) |
622 | && CPU_FEATURE_USABLE (RTM)), |
623 | __strncasecmp_l_avx2_rtm) |
624 | IFUNC_IMPL_ADD (array, i, strncasecmp_l, |
625 | CPU_FEATURE_USABLE (SSE4_2), |
626 | __strncasecmp_l_sse42) |
627 | IFUNC_IMPL_ADD (array, i, strncasecmp_l, |
628 | CPU_FEATURE_USABLE (SSSE3), |
629 | __strncasecmp_l_ssse3) |
630 | IFUNC_IMPL_ADD (array, i, strncasecmp_l, 1, |
631 | __strncasecmp_l_sse2)) |
632 | |
633 | /* Support sysdeps/x86_64/multiarch/strncat.c. */ |
634 | IFUNC_IMPL (i, name, strncat, |
635 | IFUNC_IMPL_ADD (array, i, strncat, CPU_FEATURE_USABLE (AVX2), |
636 | __strncat_avx2) |
637 | IFUNC_IMPL_ADD (array, i, strncat, |
638 | (CPU_FEATURE_USABLE (AVX2) |
639 | && CPU_FEATURE_USABLE (RTM)), |
640 | __strncat_avx2_rtm) |
641 | IFUNC_IMPL_ADD (array, i, strncat, |
642 | (CPU_FEATURE_USABLE (AVX512VL) |
643 | && CPU_FEATURE_USABLE (AVX512BW)), |
644 | __strncat_evex) |
645 | IFUNC_IMPL_ADD (array, i, strncat, CPU_FEATURE_USABLE (SSSE3), |
646 | __strncat_ssse3) |
647 | IFUNC_IMPL_ADD (array, i, strncat, 1, |
648 | __strncat_sse2_unaligned) |
649 | IFUNC_IMPL_ADD (array, i, strncat, 1, __strncat_sse2)) |
650 | |
651 | /* Support sysdeps/x86_64/multiarch/strncpy.c. */ |
652 | IFUNC_IMPL (i, name, strncpy, |
653 | IFUNC_IMPL_ADD (array, i, strncpy, CPU_FEATURE_USABLE (AVX2), |
654 | __strncpy_avx2) |
655 | IFUNC_IMPL_ADD (array, i, strncpy, |
656 | (CPU_FEATURE_USABLE (AVX2) |
657 | && CPU_FEATURE_USABLE (RTM)), |
658 | __strncpy_avx2_rtm) |
659 | IFUNC_IMPL_ADD (array, i, strncpy, |
660 | (CPU_FEATURE_USABLE (AVX512VL) |
661 | && CPU_FEATURE_USABLE (AVX512BW)), |
662 | __strncpy_evex) |
663 | IFUNC_IMPL_ADD (array, i, strncpy, CPU_FEATURE_USABLE (SSSE3), |
664 | __strncpy_ssse3) |
665 | IFUNC_IMPL_ADD (array, i, strncpy, 1, |
666 | __strncpy_sse2_unaligned) |
667 | IFUNC_IMPL_ADD (array, i, strncpy, 1, __strncpy_sse2)) |
668 | |
669 | /* Support sysdeps/x86_64/multiarch/strpbrk.c. */ |
670 | IFUNC_IMPL (i, name, strpbrk, |
671 | IFUNC_IMPL_ADD (array, i, strpbrk, CPU_FEATURE_USABLE (SSE4_2), |
672 | __strpbrk_sse42) |
673 | IFUNC_IMPL_ADD (array, i, strpbrk, 1, __strpbrk_sse2)) |
674 | |
675 | |
676 | /* Support sysdeps/x86_64/multiarch/strspn.c. */ |
677 | IFUNC_IMPL (i, name, strspn, |
678 | IFUNC_IMPL_ADD (array, i, strspn, CPU_FEATURE_USABLE (SSE4_2), |
679 | __strspn_sse42) |
680 | IFUNC_IMPL_ADD (array, i, strspn, 1, __strspn_sse2)) |
681 | |
682 | /* Support sysdeps/x86_64/multiarch/strstr.c. */ |
683 | IFUNC_IMPL (i, name, strstr, |
684 | IFUNC_IMPL_ADD (array, i, strstr, |
685 | (CPU_FEATURE_USABLE (AVX512VL) |
686 | && CPU_FEATURE_USABLE (AVX512BW) |
687 | && CPU_FEATURE_USABLE (AVX512DQ) |
688 | && CPU_FEATURE_USABLE (BMI2)), |
689 | __strstr_avx512) |
690 | IFUNC_IMPL_ADD (array, i, strstr, 1, __strstr_sse2_unaligned) |
691 | IFUNC_IMPL_ADD (array, i, strstr, 1, __strstr_sse2)) |
692 | |
693 | /* Support sysdeps/x86_64/multiarch/wcschr.c. */ |
694 | IFUNC_IMPL (i, name, wcschr, |
695 | IFUNC_IMPL_ADD (array, i, wcschr, |
696 | (CPU_FEATURE_USABLE (AVX2) |
697 | && CPU_FEATURE_USABLE (BMI2)), |
698 | __wcschr_avx2) |
699 | IFUNC_IMPL_ADD (array, i, wcschr, |
700 | (CPU_FEATURE_USABLE (AVX2) |
701 | && CPU_FEATURE_USABLE (BMI2) |
702 | && CPU_FEATURE_USABLE (RTM)), |
703 | __wcschr_avx2_rtm) |
704 | IFUNC_IMPL_ADD (array, i, wcschr, |
705 | (CPU_FEATURE_USABLE (AVX512VL) |
706 | && CPU_FEATURE_USABLE (AVX512BW) |
707 | && CPU_FEATURE_USABLE (BMI2)), |
708 | __wcschr_evex) |
709 | IFUNC_IMPL_ADD (array, i, wcschr, 1, __wcschr_sse2)) |
710 | |
711 | /* Support sysdeps/x86_64/multiarch/wcsrchr.c. */ |
712 | IFUNC_IMPL (i, name, wcsrchr, |
713 | IFUNC_IMPL_ADD (array, i, wcsrchr, |
714 | (CPU_FEATURE_USABLE (AVX2) |
715 | && CPU_FEATURE_USABLE (BMI1) |
716 | && CPU_FEATURE_USABLE (BMI2)), |
717 | __wcsrchr_avx2) |
718 | IFUNC_IMPL_ADD (array, i, wcsrchr, |
719 | (CPU_FEATURE_USABLE (AVX2) |
720 | && CPU_FEATURE_USABLE (BMI1) |
721 | && CPU_FEATURE_USABLE (BMI2) |
722 | && CPU_FEATURE_USABLE (RTM)), |
723 | __wcsrchr_avx2_rtm) |
724 | IFUNC_IMPL_ADD (array, i, wcsrchr, |
725 | (CPU_FEATURE_USABLE (AVX512VL) |
726 | && CPU_FEATURE_USABLE (AVX512BW) |
727 | && CPU_FEATURE_USABLE (BMI1) |
728 | && CPU_FEATURE_USABLE (BMI2)), |
729 | __wcsrchr_evex) |
730 | IFUNC_IMPL_ADD (array, i, wcsrchr, 1, __wcsrchr_sse2)) |
731 | |
732 | /* Support sysdeps/x86_64/multiarch/wcscmp.c. */ |
733 | IFUNC_IMPL (i, name, wcscmp, |
734 | IFUNC_IMPL_ADD (array, i, wcscmp, |
735 | (CPU_FEATURE_USABLE (AVX2) |
736 | && CPU_FEATURE_USABLE (BMI2)), |
737 | __wcscmp_avx2) |
738 | IFUNC_IMPL_ADD (array, i, wcscmp, |
739 | (CPU_FEATURE_USABLE (AVX2) |
740 | && CPU_FEATURE_USABLE (BMI2) |
741 | && CPU_FEATURE_USABLE (RTM)), |
742 | __wcscmp_avx2_rtm) |
743 | IFUNC_IMPL_ADD (array, i, wcscmp, |
744 | (CPU_FEATURE_USABLE (AVX512VL) |
745 | && CPU_FEATURE_USABLE (AVX512BW) |
746 | && CPU_FEATURE_USABLE (BMI2)), |
747 | __wcscmp_evex) |
748 | IFUNC_IMPL_ADD (array, i, wcscmp, 1, __wcscmp_sse2)) |
749 | |
750 | /* Support sysdeps/x86_64/multiarch/wcsncmp.c. */ |
751 | IFUNC_IMPL (i, name, wcsncmp, |
752 | IFUNC_IMPL_ADD (array, i, wcsncmp, |
753 | (CPU_FEATURE_USABLE (AVX2) |
754 | && CPU_FEATURE_USABLE (BMI2)), |
755 | __wcsncmp_avx2) |
756 | IFUNC_IMPL_ADD (array, i, wcsncmp, |
757 | (CPU_FEATURE_USABLE (AVX2) |
758 | && CPU_FEATURE_USABLE (BMI2) |
759 | && CPU_FEATURE_USABLE (RTM)), |
760 | __wcsncmp_avx2_rtm) |
761 | IFUNC_IMPL_ADD (array, i, wcsncmp, |
762 | (CPU_FEATURE_USABLE (AVX512VL) |
763 | && CPU_FEATURE_USABLE (AVX512BW) |
764 | && CPU_FEATURE_USABLE (BMI2)), |
765 | __wcsncmp_evex) |
766 | IFUNC_IMPL_ADD (array, i, wcsncmp, 1, __wcsncmp_sse2)) |
767 | |
768 | /* Support sysdeps/x86_64/multiarch/wcscpy.c. */ |
769 | IFUNC_IMPL (i, name, wcscpy, |
770 | IFUNC_IMPL_ADD (array, i, wcscpy, CPU_FEATURE_USABLE (SSSE3), |
771 | __wcscpy_ssse3) |
772 | IFUNC_IMPL_ADD (array, i, wcscpy, 1, __wcscpy_sse2)) |
773 | |
774 | /* Support sysdeps/x86_64/multiarch/wcslen.c. */ |
775 | IFUNC_IMPL (i, name, wcslen, |
776 | IFUNC_IMPL_ADD (array, i, wcslen, |
777 | (CPU_FEATURE_USABLE (AVX2) |
778 | && CPU_FEATURE_USABLE (BMI2)), |
779 | __wcslen_avx2) |
780 | IFUNC_IMPL_ADD (array, i, wcslen, |
781 | (CPU_FEATURE_USABLE (AVX2) |
782 | && CPU_FEATURE_USABLE (BMI2) |
783 | && CPU_FEATURE_USABLE (RTM)), |
784 | __wcslen_avx2_rtm) |
785 | IFUNC_IMPL_ADD (array, i, wcslen, |
786 | (CPU_FEATURE_USABLE (AVX512VL) |
787 | && CPU_FEATURE_USABLE (AVX512BW) |
788 | && CPU_FEATURE_USABLE (BMI2)), |
789 | __wcslen_evex) |
790 | IFUNC_IMPL_ADD (array, i, wcslen, |
791 | (CPU_FEATURE_USABLE (AVX512VL) |
792 | && CPU_FEATURE_USABLE (AVX512BW) |
793 | && CPU_FEATURE_USABLE (BMI2)), |
794 | __wcslen_evex512) |
795 | IFUNC_IMPL_ADD (array, i, wcslen, |
796 | CPU_FEATURE_USABLE (SSE4_1), |
797 | __wcslen_sse4_1) |
798 | IFUNC_IMPL_ADD (array, i, wcslen, 1, __wcslen_sse2)) |
799 | |
800 | /* Support sysdeps/x86_64/multiarch/wcsnlen.c. */ |
801 | IFUNC_IMPL (i, name, wcsnlen, |
802 | IFUNC_IMPL_ADD (array, i, wcsnlen, |
803 | (CPU_FEATURE_USABLE (AVX2) |
804 | && CPU_FEATURE_USABLE (BMI2)), |
805 | __wcsnlen_avx2) |
806 | IFUNC_IMPL_ADD (array, i, wcsnlen, |
807 | (CPU_FEATURE_USABLE (AVX2) |
808 | && CPU_FEATURE_USABLE (BMI2) |
809 | && CPU_FEATURE_USABLE (RTM)), |
810 | __wcsnlen_avx2_rtm) |
811 | IFUNC_IMPL_ADD (array, i, wcsnlen, |
812 | (CPU_FEATURE_USABLE (AVX512VL) |
813 | && CPU_FEATURE_USABLE (AVX512BW) |
814 | && CPU_FEATURE_USABLE (BMI2)), |
815 | __wcsnlen_evex) |
816 | IFUNC_IMPL_ADD (array, i, wcsnlen, |
817 | (CPU_FEATURE_USABLE (AVX512VL) |
818 | && CPU_FEATURE_USABLE (AVX512BW) |
819 | && CPU_FEATURE_USABLE (BMI2)), |
820 | __wcsnlen_evex512) |
821 | IFUNC_IMPL_ADD (array, i, wcsnlen, |
822 | CPU_FEATURE_USABLE (SSE4_1), |
823 | __wcsnlen_sse4_1) |
824 | IFUNC_IMPL_ADD (array, i, wcsnlen, 1, __wcsnlen_sse2)) |
825 | |
826 | /* Support sysdeps/x86_64/multiarch/wmemchr.c. */ |
827 | IFUNC_IMPL (i, name, wmemchr, |
828 | IFUNC_IMPL_ADD (array, i, wmemchr, |
829 | (CPU_FEATURE_USABLE (AVX2) |
830 | && CPU_FEATURE_USABLE (BMI2)), |
831 | __wmemchr_avx2) |
832 | IFUNC_IMPL_ADD (array, i, wmemchr, |
833 | (CPU_FEATURE_USABLE (AVX2) |
834 | && CPU_FEATURE_USABLE (BMI2) |
835 | && CPU_FEATURE_USABLE (RTM)), |
836 | __wmemchr_avx2_rtm) |
837 | IFUNC_IMPL_ADD (array, i, wmemchr, |
838 | (CPU_FEATURE_USABLE (AVX512VL) |
839 | && CPU_FEATURE_USABLE (AVX512BW) |
840 | && CPU_FEATURE_USABLE (BMI2)), |
841 | __wmemchr_evex) |
842 | IFUNC_IMPL_ADD (array, i, wmemchr, |
843 | (CPU_FEATURE_USABLE (AVX512VL) |
844 | && CPU_FEATURE_USABLE (AVX512BW) |
845 | && CPU_FEATURE_USABLE (BMI2)), |
846 | __wmemchr_evex_rtm) |
847 | IFUNC_IMPL_ADD (array, i, wmemchr, 1, __wmemchr_sse2)) |
848 | |
849 | /* Support sysdeps/x86_64/multiarch/wmemcmp.c. */ |
850 | IFUNC_IMPL (i, name, wmemcmp, |
851 | IFUNC_IMPL_ADD (array, i, wmemcmp, |
852 | (CPU_FEATURE_USABLE (AVX2) |
853 | && CPU_FEATURE_USABLE (BMI2) |
854 | && CPU_FEATURE_USABLE (MOVBE)), |
855 | __wmemcmp_avx2_movbe) |
856 | IFUNC_IMPL_ADD (array, i, wmemcmp, |
857 | (CPU_FEATURE_USABLE (AVX2) |
858 | && CPU_FEATURE_USABLE (BMI2) |
859 | && CPU_FEATURE_USABLE (MOVBE) |
860 | && CPU_FEATURE_USABLE (RTM)), |
861 | __wmemcmp_avx2_movbe_rtm) |
862 | IFUNC_IMPL_ADD (array, i, wmemcmp, |
863 | (CPU_FEATURE_USABLE (AVX512VL) |
864 | && CPU_FEATURE_USABLE (AVX512BW) |
865 | && CPU_FEATURE_USABLE (BMI2) |
866 | && CPU_FEATURE_USABLE (MOVBE)), |
867 | __wmemcmp_evex_movbe) |
868 | IFUNC_IMPL_ADD (array, i, wmemcmp, CPU_FEATURE_USABLE (SSSE3), |
869 | __wmemcmp_ssse3) |
870 | IFUNC_IMPL_ADD (array, i, wmemcmp, 1, __wmemcmp_sse2)) |
871 | |
872 | /* Support sysdeps/x86_64/multiarch/wmemset.c. */ |
873 | IFUNC_IMPL (i, name, wmemset, |
874 | IFUNC_IMPL_ADD (array, i, wmemset, 1, |
875 | __wmemset_sse2_unaligned) |
876 | IFUNC_IMPL_ADD (array, i, wmemset, |
877 | CPU_FEATURE_USABLE (AVX2), |
878 | __wmemset_avx2_unaligned) |
879 | IFUNC_IMPL_ADD (array, i, wmemset, |
880 | (CPU_FEATURE_USABLE (AVX2) |
881 | && CPU_FEATURE_USABLE (RTM)), |
882 | __wmemset_avx2_unaligned_rtm) |
883 | IFUNC_IMPL_ADD (array, i, wmemset, |
884 | (CPU_FEATURE_USABLE (AVX512VL) |
885 | && CPU_FEATURE_USABLE (AVX512BW) |
886 | && CPU_FEATURE_USABLE (BMI2)), |
887 | __wmemset_evex_unaligned) |
888 | IFUNC_IMPL_ADD (array, i, wmemset, |
889 | (CPU_FEATURE_USABLE (AVX512VL) |
890 | && CPU_FEATURE_USABLE (AVX512BW) |
891 | && CPU_FEATURE_USABLE (BMI2)), |
892 | __wmemset_avx512_unaligned)) |
893 | |
894 | #ifdef SHARED |
895 | /* Support sysdeps/x86_64/multiarch/memcpy_chk.c. */ |
896 | IFUNC_IMPL (i, name, __memcpy_chk, |
897 | IFUNC_IMPL_ADD (array, i, __memcpy_chk, |
898 | CPU_FEATURE_USABLE (AVX512F), |
899 | __memcpy_chk_avx512_no_vzeroupper) |
900 | IFUNC_IMPL_ADD (array, i, __memcpy_chk, |
901 | CPU_FEATURE_USABLE (AVX512VL), |
902 | __memcpy_chk_avx512_unaligned) |
903 | IFUNC_IMPL_ADD (array, i, __memcpy_chk, |
904 | CPU_FEATURE_USABLE (AVX512VL), |
905 | __memcpy_chk_avx512_unaligned_erms) |
906 | IFUNC_IMPL_ADD (array, i, __memcpy_chk, |
907 | CPU_FEATURE_USABLE (AVX), |
908 | __memcpy_chk_avx_unaligned) |
909 | IFUNC_IMPL_ADD (array, i, __memcpy_chk, |
910 | CPU_FEATURE_USABLE (AVX), |
911 | __memcpy_chk_avx_unaligned_erms) |
912 | IFUNC_IMPL_ADD (array, i, __memcpy_chk, |
913 | (CPU_FEATURE_USABLE (AVX) |
914 | && CPU_FEATURE_USABLE (RTM)), |
915 | __memcpy_chk_avx_unaligned_rtm) |
916 | IFUNC_IMPL_ADD (array, i, __memcpy_chk, |
917 | (CPU_FEATURE_USABLE (AVX) |
918 | && CPU_FEATURE_USABLE (RTM)), |
919 | __memcpy_chk_avx_unaligned_erms_rtm) |
920 | IFUNC_IMPL_ADD (array, i, __memcpy_chk, |
921 | CPU_FEATURE_USABLE (AVX512VL), |
922 | __memcpy_chk_evex_unaligned) |
923 | IFUNC_IMPL_ADD (array, i, __memcpy_chk, |
924 | CPU_FEATURE_USABLE (AVX512VL), |
925 | __memcpy_chk_evex_unaligned_erms) |
926 | IFUNC_IMPL_ADD (array, i, __memcpy_chk, |
927 | CPU_FEATURE_USABLE (SSSE3), |
928 | __memcpy_chk_ssse3_back) |
929 | IFUNC_IMPL_ADD (array, i, __memcpy_chk, |
930 | CPU_FEATURE_USABLE (SSSE3), |
931 | __memcpy_chk_ssse3) |
932 | IFUNC_IMPL_ADD (array, i, __memcpy_chk, 1, |
933 | __memcpy_chk_sse2_unaligned) |
934 | IFUNC_IMPL_ADD (array, i, __memcpy_chk, 1, |
935 | __memcpy_chk_sse2_unaligned_erms) |
936 | IFUNC_IMPL_ADD (array, i, __memcpy_chk, 1, |
937 | __memcpy_chk_erms)) |
938 | #endif |
939 | |
940 | /* Support sysdeps/x86_64/multiarch/memcpy.c. */ |
941 | IFUNC_IMPL (i, name, memcpy, |
942 | IFUNC_IMPL_ADD (array, i, memcpy, |
943 | CPU_FEATURE_USABLE (AVX), |
944 | __memcpy_avx_unaligned) |
945 | IFUNC_IMPL_ADD (array, i, memcpy, |
946 | CPU_FEATURE_USABLE (AVX), |
947 | __memcpy_avx_unaligned_erms) |
948 | IFUNC_IMPL_ADD (array, i, memcpy, |
949 | (CPU_FEATURE_USABLE (AVX) |
950 | && CPU_FEATURE_USABLE (RTM)), |
951 | __memcpy_avx_unaligned_rtm) |
952 | IFUNC_IMPL_ADD (array, i, memcpy, |
953 | (CPU_FEATURE_USABLE (AVX) |
954 | && CPU_FEATURE_USABLE (RTM)), |
955 | __memcpy_avx_unaligned_erms_rtm) |
956 | IFUNC_IMPL_ADD (array, i, memcpy, |
957 | CPU_FEATURE_USABLE (AVX512VL), |
958 | __memcpy_evex_unaligned) |
959 | IFUNC_IMPL_ADD (array, i, memcpy, |
960 | CPU_FEATURE_USABLE (AVX512VL), |
961 | __memcpy_evex_unaligned_erms) |
962 | IFUNC_IMPL_ADD (array, i, memcpy, CPU_FEATURE_USABLE (SSSE3), |
963 | __memcpy_ssse3_back) |
964 | IFUNC_IMPL_ADD (array, i, memcpy, CPU_FEATURE_USABLE (SSSE3), |
965 | __memcpy_ssse3) |
966 | IFUNC_IMPL_ADD (array, i, memcpy, |
967 | CPU_FEATURE_USABLE (AVX512F), |
968 | __memcpy_avx512_no_vzeroupper) |
969 | IFUNC_IMPL_ADD (array, i, memcpy, |
970 | CPU_FEATURE_USABLE (AVX512VL), |
971 | __memcpy_avx512_unaligned) |
972 | IFUNC_IMPL_ADD (array, i, memcpy, |
973 | CPU_FEATURE_USABLE (AVX512VL), |
974 | __memcpy_avx512_unaligned_erms) |
975 | IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_sse2_unaligned) |
976 | IFUNC_IMPL_ADD (array, i, memcpy, 1, |
977 | __memcpy_sse2_unaligned_erms) |
978 | IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_erms)) |
979 | |
980 | #ifdef SHARED |
981 | /* Support sysdeps/x86_64/multiarch/mempcpy_chk.c. */ |
982 | IFUNC_IMPL (i, name, __mempcpy_chk, |
983 | IFUNC_IMPL_ADD (array, i, __mempcpy_chk, |
984 | CPU_FEATURE_USABLE (AVX512F), |
985 | __mempcpy_chk_avx512_no_vzeroupper) |
986 | IFUNC_IMPL_ADD (array, i, __mempcpy_chk, |
987 | CPU_FEATURE_USABLE (AVX512VL), |
988 | __mempcpy_chk_avx512_unaligned) |
989 | IFUNC_IMPL_ADD (array, i, __mempcpy_chk, |
990 | CPU_FEATURE_USABLE (AVX512VL), |
991 | __mempcpy_chk_avx512_unaligned_erms) |
992 | IFUNC_IMPL_ADD (array, i, __mempcpy_chk, |
993 | CPU_FEATURE_USABLE (AVX), |
994 | __mempcpy_chk_avx_unaligned) |
995 | IFUNC_IMPL_ADD (array, i, __mempcpy_chk, |
996 | CPU_FEATURE_USABLE (AVX), |
997 | __mempcpy_chk_avx_unaligned_erms) |
998 | IFUNC_IMPL_ADD (array, i, __mempcpy_chk, |
999 | (CPU_FEATURE_USABLE (AVX) |
1000 | && CPU_FEATURE_USABLE (RTM)), |
1001 | __mempcpy_chk_avx_unaligned_rtm) |
1002 | IFUNC_IMPL_ADD (array, i, __mempcpy_chk, |
1003 | (CPU_FEATURE_USABLE (AVX) |
1004 | && CPU_FEATURE_USABLE (RTM)), |
1005 | __mempcpy_chk_avx_unaligned_erms_rtm) |
1006 | IFUNC_IMPL_ADD (array, i, __mempcpy_chk, |
1007 | CPU_FEATURE_USABLE (AVX512VL), |
1008 | __mempcpy_chk_evex_unaligned) |
1009 | IFUNC_IMPL_ADD (array, i, __mempcpy_chk, |
1010 | CPU_FEATURE_USABLE (AVX512VL), |
1011 | __mempcpy_chk_evex_unaligned_erms) |
1012 | IFUNC_IMPL_ADD (array, i, __mempcpy_chk, |
1013 | CPU_FEATURE_USABLE (SSSE3), |
1014 | __mempcpy_chk_ssse3_back) |
1015 | IFUNC_IMPL_ADD (array, i, __mempcpy_chk, |
1016 | CPU_FEATURE_USABLE (SSSE3), |
1017 | __mempcpy_chk_ssse3) |
1018 | IFUNC_IMPL_ADD (array, i, __mempcpy_chk, 1, |
1019 | __mempcpy_chk_sse2_unaligned) |
1020 | IFUNC_IMPL_ADD (array, i, __mempcpy_chk, 1, |
1021 | __mempcpy_chk_sse2_unaligned_erms) |
1022 | IFUNC_IMPL_ADD (array, i, __mempcpy_chk, 1, |
1023 | __mempcpy_chk_erms)) |
1024 | #endif |
1025 | |
1026 | /* Support sysdeps/x86_64/multiarch/mempcpy.c. */ |
1027 | IFUNC_IMPL (i, name, mempcpy, |
1028 | IFUNC_IMPL_ADD (array, i, mempcpy, |
1029 | CPU_FEATURE_USABLE (AVX512F), |
1030 | __mempcpy_avx512_no_vzeroupper) |
1031 | IFUNC_IMPL_ADD (array, i, mempcpy, |
1032 | CPU_FEATURE_USABLE (AVX512VL), |
1033 | __mempcpy_avx512_unaligned) |
1034 | IFUNC_IMPL_ADD (array, i, mempcpy, |
1035 | CPU_FEATURE_USABLE (AVX512VL), |
1036 | __mempcpy_avx512_unaligned_erms) |
1037 | IFUNC_IMPL_ADD (array, i, mempcpy, |
1038 | CPU_FEATURE_USABLE (AVX), |
1039 | __mempcpy_avx_unaligned) |
1040 | IFUNC_IMPL_ADD (array, i, mempcpy, |
1041 | CPU_FEATURE_USABLE (AVX), |
1042 | __mempcpy_avx_unaligned_erms) |
1043 | IFUNC_IMPL_ADD (array, i, mempcpy, |
1044 | (CPU_FEATURE_USABLE (AVX) |
1045 | && CPU_FEATURE_USABLE (RTM)), |
1046 | __mempcpy_avx_unaligned_rtm) |
1047 | IFUNC_IMPL_ADD (array, i, mempcpy, |
1048 | (CPU_FEATURE_USABLE (AVX) |
1049 | && CPU_FEATURE_USABLE (RTM)), |
1050 | __mempcpy_avx_unaligned_erms_rtm) |
1051 | IFUNC_IMPL_ADD (array, i, mempcpy, |
1052 | CPU_FEATURE_USABLE (AVX512VL), |
1053 | __mempcpy_evex_unaligned) |
1054 | IFUNC_IMPL_ADD (array, i, mempcpy, |
1055 | CPU_FEATURE_USABLE (AVX512VL), |
1056 | __mempcpy_evex_unaligned_erms) |
1057 | IFUNC_IMPL_ADD (array, i, mempcpy, CPU_FEATURE_USABLE (SSSE3), |
1058 | __mempcpy_ssse3_back) |
1059 | IFUNC_IMPL_ADD (array, i, mempcpy, CPU_FEATURE_USABLE (SSSE3), |
1060 | __mempcpy_ssse3) |
1061 | IFUNC_IMPL_ADD (array, i, mempcpy, 1, |
1062 | __mempcpy_sse2_unaligned) |
1063 | IFUNC_IMPL_ADD (array, i, mempcpy, 1, |
1064 | __mempcpy_sse2_unaligned_erms) |
1065 | IFUNC_IMPL_ADD (array, i, mempcpy, 1, __mempcpy_erms)) |
1066 | |
1067 | /* Support sysdeps/x86_64/multiarch/strncmp.c. */ |
1068 | IFUNC_IMPL (i, name, strncmp, |
1069 | IFUNC_IMPL_ADD (array, i, strncmp, |
1070 | (CPU_FEATURE_USABLE (AVX2) |
1071 | && CPU_FEATURE_USABLE (BMI2)), |
1072 | __strncmp_avx2) |
1073 | IFUNC_IMPL_ADD (array, i, strncmp, |
1074 | (CPU_FEATURE_USABLE (AVX2) |
1075 | && CPU_FEATURE_USABLE (BMI2) |
1076 | && CPU_FEATURE_USABLE (RTM)), |
1077 | __strncmp_avx2_rtm) |
1078 | IFUNC_IMPL_ADD (array, i, strncmp, |
1079 | (CPU_FEATURE_USABLE (AVX512VL) |
1080 | && CPU_FEATURE_USABLE (AVX512BW) |
1081 | && CPU_FEATURE_USABLE (BMI2)), |
1082 | __strncmp_evex) |
1083 | IFUNC_IMPL_ADD (array, i, strncmp, CPU_FEATURE_USABLE (SSE4_2), |
1084 | __strncmp_sse42) |
1085 | IFUNC_IMPL_ADD (array, i, strncmp, CPU_FEATURE_USABLE (SSSE3), |
1086 | __strncmp_ssse3) |
1087 | IFUNC_IMPL_ADD (array, i, strncmp, 1, __strncmp_sse2)) |
1088 | |
1089 | #ifdef SHARED |
1090 | /* Support sysdeps/x86_64/multiarch/wmemset_chk.c. */ |
1091 | IFUNC_IMPL (i, name, __wmemset_chk, |
1092 | IFUNC_IMPL_ADD (array, i, __wmemset_chk, 1, |
1093 | __wmemset_chk_sse2_unaligned) |
1094 | IFUNC_IMPL_ADD (array, i, __wmemset_chk, |
1095 | CPU_FEATURE_USABLE (AVX2), |
1096 | __wmemset_chk_avx2_unaligned) |
1097 | IFUNC_IMPL_ADD (array, i, __wmemset_chk, |
1098 | (CPU_FEATURE_USABLE (AVX2) |
1099 | && CPU_FEATURE_USABLE (RTM)), |
1100 | __wmemset_chk_avx2_unaligned_rtm) |
1101 | IFUNC_IMPL_ADD (array, i, __wmemset_chk, |
1102 | (CPU_FEATURE_USABLE (AVX512VL) |
1103 | && CPU_FEATURE_USABLE (AVX512BW) |
1104 | && CPU_FEATURE_USABLE (BMI2)), |
1105 | __wmemset_chk_evex_unaligned) |
1106 | IFUNC_IMPL_ADD (array, i, __wmemset_chk, |
1107 | (CPU_FEATURE_USABLE (AVX512VL) |
1108 | && CPU_FEATURE_USABLE (AVX512BW) |
1109 | && CPU_FEATURE_USABLE (BMI2)), |
1110 | __wmemset_chk_avx512_unaligned)) |
1111 | #endif |
1112 | |
1113 | return i; |
1114 | } |
1115 | |