1/* Copyright (C) 2012-2022 Free Software Foundation, Inc.
2
3 This file is part of the GNU C Library.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library. If not, see
17 <https://www.gnu.org/licenses/>. */
18
19#include <sysdep.h>
20#include "memset-reg.h"
21
22#ifndef MEMSET
23# define MEMSET memset
24#endif
25
26/* Assumptions:
27 *
28 * ARMv8-a, AArch64, unaligned accesses
29 *
30 */
31
32ENTRY_ALIGN (MEMSET, 6)
33
34 PTR_ARG (0)
35 SIZE_ARG (2)
36
37 dup v0.16B, valw
38 add dstend, dstin, count
39
40 cmp count, 96
41 b.hi L(set_long)
42 cmp count, 16
43 b.hs L(set_medium)
44 mov val, v0.D[0]
45
46 /* Set 0..15 bytes. */
47 tbz count, 3, 1f
48 str val, [dstin]
49 str val, [dstend, -8]
50 ret
51 nop
521: tbz count, 2, 2f
53 str valw, [dstin]
54 str valw, [dstend, -4]
55 ret
562: cbz count, 3f
57 strb valw, [dstin]
58 tbz count, 1, 3f
59 strh valw, [dstend, -2]
603: ret
61
62 /* Set 17..96 bytes. */
63L(set_medium):
64 str q0, [dstin]
65 tbnz count, 6, L(set96)
66 str q0, [dstend, -16]
67 tbz count, 5, 1f
68 str q0, [dstin, 16]
69 str q0, [dstend, -32]
701: ret
71
72 .p2align 4
73 /* Set 64..96 bytes. Write 64 bytes from the start and
74 32 bytes from the end. */
75L(set96):
76 str q0, [dstin, 16]
77 stp q0, q0, [dstin, 32]
78 stp q0, q0, [dstend, -32]
79 ret
80
81 .p2align 3
82 nop
83L(set_long):
84 and valw, valw, 255
85 bic dst, dstin, 15
86 str q0, [dstin]
87 cmp count, 256
88 ccmp valw, 0, 0, cs
89 b.eq L(try_zva)
90L(no_zva):
91 sub count, dstend, dst /* Count is 16 too large. */
92 sub dst, dst, 16 /* Dst is biased by -32. */
93 sub count, count, 64 + 16 /* Adjust count and bias for loop. */
941: stp q0, q0, [dst, 32]
95 stp q0, q0, [dst, 64]!
96L(tail64):
97 subs count, count, 64
98 b.hi 1b
992: stp q0, q0, [dstend, -64]
100 stp q0, q0, [dstend, -32]
101 ret
102
103L(try_zva):
104#ifdef ZVA_MACRO
105 zva_macro
106#else
107 .p2align 3
108 mrs tmp1, dczid_el0
109 tbnz tmp1w, 4, L(no_zva)
110 and tmp1w, tmp1w, 15
111 cmp tmp1w, 4 /* ZVA size is 64 bytes. */
112 b.ne L(zva_128)
113
114 /* Write the first and last 64 byte aligned block using stp rather
115 than using DC ZVA. This is faster on some cores.
116 */
117L(zva_64):
118 str q0, [dst, 16]
119 stp q0, q0, [dst, 32]
120 bic dst, dst, 63
121 stp q0, q0, [dst, 64]
122 stp q0, q0, [dst, 96]
123 sub count, dstend, dst /* Count is now 128 too large. */
124 sub count, count, 128+64+64 /* Adjust count and bias for loop. */
125 add dst, dst, 128
126 nop
1271: dc zva, dst
128 add dst, dst, 64
129 subs count, count, 64
130 b.hi 1b
131 stp q0, q0, [dst, 0]
132 stp q0, q0, [dst, 32]
133 stp q0, q0, [dstend, -64]
134 stp q0, q0, [dstend, -32]
135 ret
136
137 .p2align 3
138L(zva_128):
139 cmp tmp1w, 5 /* ZVA size is 128 bytes. */
140 b.ne L(zva_other)
141
142 str q0, [dst, 16]
143 stp q0, q0, [dst, 32]
144 stp q0, q0, [dst, 64]
145 stp q0, q0, [dst, 96]
146 bic dst, dst, 127
147 sub count, dstend, dst /* Count is now 128 too large. */
148 sub count, count, 128+128 /* Adjust count and bias for loop. */
149 add dst, dst, 128
1501: dc zva, dst
151 add dst, dst, 128
152 subs count, count, 128
153 b.hi 1b
154 stp q0, q0, [dstend, -128]
155 stp q0, q0, [dstend, -96]
156 stp q0, q0, [dstend, -64]
157 stp q0, q0, [dstend, -32]
158 ret
159
160L(zva_other):
161 mov tmp2w, 4
162 lsl zva_lenw, tmp2w, tmp1w
163 add tmp1, zva_len, 64 /* Max alignment bytes written. */
164 cmp count, tmp1
165 blo L(no_zva)
166
167 sub tmp2, zva_len, 1
168 add tmp1, dst, zva_len
169 add dst, dst, 16
170 subs count, tmp1, dst /* Actual alignment bytes to write. */
171 bic tmp1, tmp1, tmp2 /* Aligned dc zva start address. */
172 beq 2f
1731: stp q0, q0, [dst], 64
174 stp q0, q0, [dst, -32]
175 subs count, count, 64
176 b.hi 1b
1772: mov dst, tmp1
178 sub count, dstend, tmp1 /* Remaining bytes to write. */
179 subs count, count, zva_len
180 b.lo 4f
1813: dc zva, dst
182 add dst, dst, zva_len
183 subs count, count, zva_len
184 b.hs 3b
1854: add count, count, zva_len
186 sub dst, dst, 32 /* Bias dst for tail loop. */
187 b L(tail64)
188#endif
189
190END (MEMSET)
191libc_hidden_builtin_def (MEMSET)
192

source code of glibc/sysdeps/aarch64/memset.S