1/* Optimized memset implementation for PowerPC.
2 Copyright (C) 1997-2024 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
18
19#include <sysdep.h>
20#include <rtld-global-offsets.h>
21
22/* void * [r3] memset (void *s [r3], int c [r4], size_t n [r5]));
23 Returns 's'.
24
25 The memset is done in four sizes: byte (8 bits), word (32 bits),
26 32-byte blocks (256 bits) and cache line size (128, 256, 1024 bits).
27 There is a special case for setting whole cache lines to 0, which
28 takes advantage of the dcbz instruction. */
29
30 .section ".text"
31EALIGN (memset, 5, 1)
32
33#define rTMP r0
34#define rRTN r3 /* initial value of 1st argument */
35#define rMEMP0 r3 /* original value of 1st arg */
36#define rCHR r4 /* char to set in each byte */
37#define rLEN r5 /* length of region to set */
38#define rMEMP r6 /* address at which we are storing */
39#define rALIGN r7 /* number of bytes we are setting now (when aligning) */
40#define rMEMP2 r8
41
42#define rPOS32 r7 /* constant +32 for clearing with dcbz */
43#define rNEG64 r8 /* constant -64 for clearing with dcbz */
44#define rNEG32 r9 /* constant -32 for clearing with dcbz */
45
46#define rGOT r9 /* Address of the Global Offset Table. */
47#define rCLS r8 /* Cache line size obtained from static. */
48#define rCLM r9 /* Cache line size mask to check for cache alignment. */
49
50/* take care of case for size <= 4 */
51 cmplwi cr1, rLEN, 4
52 andi. rALIGN, rMEMP0, 3
53 mr rMEMP, rMEMP0
54 ble- cr1, L(small)
55/* align to word boundary */
56 cmplwi cr5, rLEN, 31
57 rlwimi rCHR, rCHR, 8, 16, 23
58 beq+ L(aligned) /* 8th instruction from .align */
59 mtcrf 0x01, rMEMP0
60 subfic rALIGN, rALIGN, 4
61 add rMEMP, rMEMP, rALIGN
62 sub rLEN, rLEN, rALIGN
63 bf+ 31, L(g0)
64 stb rCHR, 0(rMEMP0)
65 bt 30, L(aligned)
66L(g0): sth rCHR, -2(rMEMP) /* 16th instruction from .align */
67/* take care of case for size < 31 */
68L(aligned):
69 mtcrf 0x01, rLEN
70 rlwimi rCHR, rCHR, 16, 0, 15
71 ble cr5, L(medium)
72/* align to cache line boundary... */
73 andi. rALIGN, rMEMP, 0x1C
74 subfic rALIGN, rALIGN, 0x20
75 beq L(caligned)
76 mtcrf 0x01, rALIGN
77 add rMEMP, rMEMP, rALIGN
78 sub rLEN, rLEN, rALIGN
79 cmplwi cr1, rALIGN, 0x10
80 mr rMEMP2, rMEMP
81 bf 28, L(a1)
82 stw rCHR, -4(rMEMP2)
83 stwu rCHR, -8(rMEMP2)
84L(a1): blt cr1, L(a2)
85 stw rCHR, -4(rMEMP2) /* 32nd instruction from .align */
86 stw rCHR, -8(rMEMP2)
87 stw rCHR, -12(rMEMP2)
88 stwu rCHR, -16(rMEMP2)
89L(a2): bf 29, L(caligned)
90 stw rCHR, -4(rMEMP2)
91/* now aligned to a cache line. */
92L(caligned):
93 cmplwi cr1, rCHR, 0
94 clrrwi. rALIGN, rLEN, 5
95 mtcrf 0x01, rLEN /* 40th instruction from .align */
96
97/* Check if we can use the special case for clearing memory using dcbz.
98 This requires that we know the correct cache line size for this
99 processor. Getting the cache line size may require establishing GOT
100 addressability, so branch out of line to set this up. */
101 beq cr1, L(checklinesize)
102
103/* Store blocks of 32-bytes (256-bits) starting on a 32-byte boundary.
104 Can't assume that rCHR is zero or that the cache line size is either
105 32-bytes or even known. */
106L(nondcbz):
107 srwi rTMP, rALIGN, 5
108 mtctr rTMP
109 beq L(medium) /* we may not actually get to do a full line */
110 clrlwi. rLEN, rLEN, 27
111 add rMEMP, rMEMP, rALIGN
112 li rNEG64, -0x40
113 bdz L(cloopdone) /* 48th instruction from .align */
114
115/* We can't use dcbz here as we don't know the cache line size. We can
116 use "data cache block touch for store", which is safe. */
117L(c3): dcbtst rNEG64, rMEMP
118 stw rCHR, -4(rMEMP)
119 stw rCHR, -8(rMEMP)
120 stw rCHR, -12(rMEMP)
121 stw rCHR, -16(rMEMP)
122 nop /* let 601 fetch last 4 instructions of loop */
123 stw rCHR, -20(rMEMP)
124 stw rCHR, -24(rMEMP) /* 56th instruction from .align */
125 nop /* let 601 fetch first 8 instructions of loop */
126 stw rCHR, -28(rMEMP)
127 stwu rCHR, -32(rMEMP)
128 bdnz L(c3)
129L(cloopdone):
130 stw rCHR, -4(rMEMP)
131 stw rCHR, -8(rMEMP)
132 stw rCHR, -12(rMEMP)
133 stw rCHR, -16(rMEMP) /* 64th instruction from .align */
134 stw rCHR, -20(rMEMP)
135 cmplwi cr1, rLEN, 16
136 stw rCHR, -24(rMEMP)
137 stw rCHR, -28(rMEMP)
138 stwu rCHR, -32(rMEMP)
139 beqlr
140 add rMEMP, rMEMP, rALIGN
141 b L(medium_tail2) /* 72nd instruction from .align */
142
143 .align 5
144 nop
145/* Clear cache lines of memory in 128-byte chunks.
146 This code is optimized for processors with 32-byte cache lines.
147 It is further optimized for the 601 processor, which requires
148 some care in how the code is aligned in the i-cache. */
149L(zloopstart):
150 clrlwi rLEN, rLEN, 27
151 mtcrf 0x02, rALIGN
152 srwi. rTMP, rALIGN, 7
153 mtctr rTMP
154 li rPOS32, 0x20
155 li rNEG64, -0x40
156 cmplwi cr1, rLEN, 16 /* 8 */
157 bf 26, L(z0)
158 dcbz 0, rMEMP
159 addi rMEMP, rMEMP, 0x20
160L(z0): li rNEG32, -0x20
161 bf 25, L(z1)
162 dcbz 0, rMEMP
163 dcbz rPOS32, rMEMP
164 addi rMEMP, rMEMP, 0x40 /* 16 */
165L(z1): cmplwi cr5, rLEN, 0
166 beq L(medium)
167L(zloop):
168 dcbz 0, rMEMP
169 dcbz rPOS32, rMEMP
170 addi rMEMP, rMEMP, 0x80
171 dcbz rNEG64, rMEMP
172 dcbz rNEG32, rMEMP
173 bdnz L(zloop)
174 beqlr cr5
175 b L(medium_tail2)
176
177 .align 5
178L(small):
179/* Memset of 4 bytes or less. */
180 cmplwi cr5, rLEN, 1
181 cmplwi cr1, rLEN, 3
182 bltlr cr5
183 stb rCHR, 0(rMEMP)
184 beqlr cr5
185 nop
186 stb rCHR, 1(rMEMP)
187 bltlr cr1
188 stb rCHR, 2(rMEMP)
189 beqlr cr1
190 nop
191 stb rCHR, 3(rMEMP)
192 blr
193
194/* Memset of 0-31 bytes. */
195 .align 5
196L(medium):
197 cmplwi cr1, rLEN, 16
198L(medium_tail2):
199 add rMEMP, rMEMP, rLEN
200L(medium_tail):
201 bt- 31, L(medium_31t)
202 bt- 30, L(medium_30t)
203L(medium_30f):
204 bt- 29, L(medium_29t)
205L(medium_29f):
206 bge- cr1, L(medium_27t)
207 bflr- 28
208 stw rCHR, -4(rMEMP) /* 8th instruction from .align */
209 stw rCHR, -8(rMEMP)
210 blr
211
212L(medium_31t):
213 stbu rCHR, -1(rMEMP)
214 bf- 30, L(medium_30f)
215L(medium_30t):
216 sthu rCHR, -2(rMEMP)
217 bf- 29, L(medium_29f)
218L(medium_29t):
219 stwu rCHR, -4(rMEMP)
220 blt- cr1, L(medium_27f) /* 16th instruction from .align */
221L(medium_27t):
222 stw rCHR, -4(rMEMP)
223 stw rCHR, -8(rMEMP)
224 stw rCHR, -12(rMEMP)
225 stwu rCHR, -16(rMEMP)
226L(medium_27f):
227 bflr- 28
228L(medium_28t):
229 stw rCHR, -4(rMEMP)
230 stw rCHR, -8(rMEMP)
231 blr
232
233L(checklinesize):
234/* If the remaining length is less the 32 bytes then don't bother getting
235 the cache line size. */
236 beq L(medium)
237#ifdef PIC
238 mflr rTMP
239/* Establishes GOT addressability so we can load the cache line size
240 from rtld_global_ro. This value was set from the aux vector during
241 startup. */
242 SETUP_GOT_ACCESS(rGOT,got_label)
243 addis rGOT,rGOT,_GLOBAL_OFFSET_TABLE_-got_label@ha
244 addi rGOT,rGOT,_GLOBAL_OFFSET_TABLE_-got_label@l
245 mtlr rTMP
246#endif
247/* Load rtld_global_ro._dl_cache_line_size. */
248 __GLRO(rCLS, rGOT, _dl_cache_line_size,
249 RTLD_GLOBAL_RO_DL_CACHE_LINE_SIZE_OFFSET)
250
251/* If the cache line size was not set then goto to L(nondcbz), which is
252 safe for any cache line size. */
253 cmplwi cr1,rCLS,0
254 beq cr1,L(nondcbz)
255
256/* If the cache line size is 32 bytes then goto to L(zloopstart),
257 which is coded specifically for 32-byte lines (and 601). */
258 cmplwi cr1,rCLS,32
259 beq cr1,L(zloopstart)
260
261/* Now we know the cache line size and it is not 32-bytes. However
262 we may not yet be aligned to the cache line and may have a partial
263 line to fill. Touch it 1st to fetch the cache line. */
264 dcbtst 0,rMEMP
265
266 addi rCLM,rCLS,-1
267L(getCacheAligned):
268 cmplwi cr1,rLEN,32
269 and. rTMP,rCLM,rMEMP
270 blt cr1,L(handletail32)
271 beq L(cacheAligned)
272/* We are not aligned to start of a cache line yet. Store 32-byte
273 of data and test again. */
274 addi rMEMP,rMEMP,32
275 addi rLEN,rLEN,-32
276 stw rCHR,-32(rMEMP)
277 stw rCHR,-28(rMEMP)
278 stw rCHR,-24(rMEMP)
279 stw rCHR,-20(rMEMP)
280 stw rCHR,-16(rMEMP)
281 stw rCHR,-12(rMEMP)
282 stw rCHR,-8(rMEMP)
283 stw rCHR,-4(rMEMP)
284 b L(getCacheAligned)
285
286/* Now we are aligned to the cache line and can use dcbz. */
287L(cacheAligned):
288 cmplw cr1,rLEN,rCLS
289 blt cr1,L(handletail32)
290 dcbz 0,rMEMP
291 subf rLEN,rCLS,rLEN
292 add rMEMP,rMEMP,rCLS
293 b L(cacheAligned)
294
295/* We are here because; the cache line size was set, it was not
296 32-bytes, and the remainder (rLEN) is now less than the actual cache
297 line size. Set up the preconditions for L(nondcbz) and go there to
298 store the remaining bytes. */
299L(handletail32):
300 clrrwi. rALIGN, rLEN, 5
301 b L(nondcbz)
302
303END (memset)
304libc_hidden_builtin_def (memset)
305

source code of glibc/sysdeps/powerpc/powerpc32/memset.S