Warning: This file is not a C or C++ file. It does not have highlighting.
1 | /* Copyright (C) 2003-2022 Free Software Foundation, Inc. |
---|---|
2 | This file is part of the GNU C Library. |
3 | |
4 | The GNU C Library is free software; you can redistribute it and/or |
5 | modify it under the terms of the GNU Lesser General Public |
6 | License as published by the Free Software Foundation; either |
7 | version 2.1 of the License, or (at your option) any later version. |
8 | |
9 | The GNU C Library is distributed in the hope that it will be useful, |
10 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
12 | Lesser General Public License for more details. |
13 | |
14 | You should have received a copy of the GNU Lesser General Public |
15 | License along with the GNU C Library. If not, see |
16 | <https://www.gnu.org/licenses/>. */ |
17 | |
18 | #include <stdint.h> |
19 | |
20 | #define __HAVE_64B_ATOMICS 1 |
21 | #define USE_ATOMIC_COMPILER_BUILTINS 0 |
22 | |
23 | /* XXX Is this actually correct? */ |
24 | #define ATOMIC_EXCHANGE_USES_CAS 1 |
25 | |
26 | |
27 | #define __MB " mb\n" |
28 | |
29 | |
30 | /* Compare and exchange. For all of the "xxx" routines, we expect a |
31 | "__prev" and a "__cmp" variable to be provided by the enclosing scope, |
32 | in which values are returned. */ |
33 | |
34 | #define __arch_compare_and_exchange_xxx_8_int(mem, new, old, mb1, mb2) \ |
35 | ({ \ |
36 | unsigned long __tmp, __snew, __addr64; \ |
37 | __asm__ __volatile__ ( \ |
38 | mb1 \ |
39 | " andnot %[__addr8],7,%[__addr64]\n" \ |
40 | " insbl %[__new],%[__addr8],%[__snew]\n" \ |
41 | "1: ldq_l %[__tmp],0(%[__addr64])\n" \ |
42 | " extbl %[__tmp],%[__addr8],%[__prev]\n" \ |
43 | " cmpeq %[__prev],%[__old],%[__cmp]\n" \ |
44 | " beq %[__cmp],2f\n" \ |
45 | " mskbl %[__tmp],%[__addr8],%[__tmp]\n" \ |
46 | " or %[__snew],%[__tmp],%[__tmp]\n" \ |
47 | " stq_c %[__tmp],0(%[__addr64])\n" \ |
48 | " beq %[__tmp],1b\n" \ |
49 | mb2 \ |
50 | "2:" \ |
51 | : [__prev] "=&r" (__prev), \ |
52 | [__snew] "=&r" (__snew), \ |
53 | [__tmp] "=&r" (__tmp), \ |
54 | [__cmp] "=&r" (__cmp), \ |
55 | [__addr64] "=&r" (__addr64) \ |
56 | : [__addr8] "r" (mem), \ |
57 | [__old] "Ir" ((uint64_t)(uint8_t)(uint64_t)(old)), \ |
58 | [__new] "r" (new) \ |
59 | : "memory"); \ |
60 | }) |
61 | |
62 | #define __arch_compare_and_exchange_xxx_16_int(mem, new, old, mb1, mb2) \ |
63 | ({ \ |
64 | unsigned long __tmp, __snew, __addr64; \ |
65 | __asm__ __volatile__ ( \ |
66 | mb1 \ |
67 | " andnot %[__addr16],7,%[__addr64]\n" \ |
68 | " inswl %[__new],%[__addr16],%[__snew]\n" \ |
69 | "1: ldq_l %[__tmp],0(%[__addr64])\n" \ |
70 | " extwl %[__tmp],%[__addr16],%[__prev]\n" \ |
71 | " cmpeq %[__prev],%[__old],%[__cmp]\n" \ |
72 | " beq %[__cmp],2f\n" \ |
73 | " mskwl %[__tmp],%[__addr16],%[__tmp]\n" \ |
74 | " or %[__snew],%[__tmp],%[__tmp]\n" \ |
75 | " stq_c %[__tmp],0(%[__addr64])\n" \ |
76 | " beq %[__tmp],1b\n" \ |
77 | mb2 \ |
78 | "2:" \ |
79 | : [__prev] "=&r" (__prev), \ |
80 | [__snew] "=&r" (__snew), \ |
81 | [__tmp] "=&r" (__tmp), \ |
82 | [__cmp] "=&r" (__cmp), \ |
83 | [__addr64] "=&r" (__addr64) \ |
84 | : [__addr16] "r" (mem), \ |
85 | [__old] "Ir" ((uint64_t)(uint16_t)(uint64_t)(old)), \ |
86 | [__new] "r" (new) \ |
87 | : "memory"); \ |
88 | }) |
89 | |
90 | #define __arch_compare_and_exchange_xxx_32_int(mem, new, old, mb1, mb2) \ |
91 | ({ \ |
92 | __asm__ __volatile__ ( \ |
93 | mb1 \ |
94 | "1: ldl_l %[__prev],%[__mem]\n" \ |
95 | " cmpeq %[__prev],%[__old],%[__cmp]\n" \ |
96 | " beq %[__cmp],2f\n" \ |
97 | " mov %[__new],%[__cmp]\n" \ |
98 | " stl_c %[__cmp],%[__mem]\n" \ |
99 | " beq %[__cmp],1b\n" \ |
100 | mb2 \ |
101 | "2:" \ |
102 | : [__prev] "=&r" (__prev), \ |
103 | [__cmp] "=&r" (__cmp) \ |
104 | : [__mem] "m" (*(mem)), \ |
105 | [__old] "Ir" ((uint64_t)(int32_t)(uint64_t)(old)), \ |
106 | [__new] "Ir" (new) \ |
107 | : "memory"); \ |
108 | }) |
109 | |
110 | #define __arch_compare_and_exchange_xxx_64_int(mem, new, old, mb1, mb2) \ |
111 | ({ \ |
112 | __asm__ __volatile__ ( \ |
113 | mb1 \ |
114 | "1: ldq_l %[__prev],%[__mem]\n" \ |
115 | " cmpeq %[__prev],%[__old],%[__cmp]\n" \ |
116 | " beq %[__cmp],2f\n" \ |
117 | " mov %[__new],%[__cmp]\n" \ |
118 | " stq_c %[__cmp],%[__mem]\n" \ |
119 | " beq %[__cmp],1b\n" \ |
120 | mb2 \ |
121 | "2:" \ |
122 | : [__prev] "=&r" (__prev), \ |
123 | [__cmp] "=&r" (__cmp) \ |
124 | : [__mem] "m" (*(mem)), \ |
125 | [__old] "Ir" ((uint64_t)(old)), \ |
126 | [__new] "Ir" (new) \ |
127 | : "memory"); \ |
128 | }) |
129 | |
130 | /* For all "bool" routines, we return FALSE if exchange succesful. */ |
131 | |
132 | #define __arch_compare_and_exchange_bool_8_int(mem, new, old, mb1, mb2) \ |
133 | ({ unsigned long __prev; int __cmp; \ |
134 | __arch_compare_and_exchange_xxx_8_int(mem, new, old, mb1, mb2); \ |
135 | !__cmp; }) |
136 | |
137 | #define __arch_compare_and_exchange_bool_16_int(mem, new, old, mb1, mb2) \ |
138 | ({ unsigned long __prev; int __cmp; \ |
139 | __arch_compare_and_exchange_xxx_16_int(mem, new, old, mb1, mb2); \ |
140 | !__cmp; }) |
141 | |
142 | #define __arch_compare_and_exchange_bool_32_int(mem, new, old, mb1, mb2) \ |
143 | ({ unsigned long __prev; int __cmp; \ |
144 | __arch_compare_and_exchange_xxx_32_int(mem, new, old, mb1, mb2); \ |
145 | !__cmp; }) |
146 | |
147 | #define __arch_compare_and_exchange_bool_64_int(mem, new, old, mb1, mb2) \ |
148 | ({ unsigned long __prev; int __cmp; \ |
149 | __arch_compare_and_exchange_xxx_64_int(mem, new, old, mb1, mb2); \ |
150 | !__cmp; }) |
151 | |
152 | /* For all "val" routines, return the old value whether exchange |
153 | successful or not. */ |
154 | |
155 | #define __arch_compare_and_exchange_val_8_int(mem, new, old, mb1, mb2) \ |
156 | ({ unsigned long __prev; int __cmp; \ |
157 | __arch_compare_and_exchange_xxx_8_int(mem, new, old, mb1, mb2); \ |
158 | (typeof (*mem))__prev; }) |
159 | |
160 | #define __arch_compare_and_exchange_val_16_int(mem, new, old, mb1, mb2) \ |
161 | ({ unsigned long __prev; int __cmp; \ |
162 | __arch_compare_and_exchange_xxx_16_int(mem, new, old, mb1, mb2); \ |
163 | (typeof (*mem))__prev; }) |
164 | |
165 | #define __arch_compare_and_exchange_val_32_int(mem, new, old, mb1, mb2) \ |
166 | ({ unsigned long __prev; int __cmp; \ |
167 | __arch_compare_and_exchange_xxx_32_int(mem, new, old, mb1, mb2); \ |
168 | (typeof (*mem))__prev; }) |
169 | |
170 | #define __arch_compare_and_exchange_val_64_int(mem, new, old, mb1, mb2) \ |
171 | ({ unsigned long __prev; int __cmp; \ |
172 | __arch_compare_and_exchange_xxx_64_int(mem, new, old, mb1, mb2); \ |
173 | (typeof (*mem))__prev; }) |
174 | |
175 | /* Compare and exchange with "acquire" semantics, ie barrier after. */ |
176 | |
177 | #define atomic_compare_and_exchange_bool_acq(mem, new, old) \ |
178 | __atomic_bool_bysize (__arch_compare_and_exchange_bool, int, \ |
179 | mem, new, old, "", __MB) |
180 | |
181 | #define atomic_compare_and_exchange_val_acq(mem, new, old) \ |
182 | __atomic_val_bysize (__arch_compare_and_exchange_val, int, \ |
183 | mem, new, old, "", __MB) |
184 | |
185 | /* Compare and exchange with "release" semantics, ie barrier before. */ |
186 | |
187 | #define atomic_compare_and_exchange_val_rel(mem, new, old) \ |
188 | __atomic_val_bysize (__arch_compare_and_exchange_val, int, \ |
189 | mem, new, old, __MB, "") |
190 | |
191 | |
192 | /* Atomically store value and return the previous value. */ |
193 | |
194 | #define __arch_exchange_8_int(mem, value, mb1, mb2) \ |
195 | ({ \ |
196 | unsigned long __tmp, __addr64, __sval; __typeof(*mem) __ret; \ |
197 | __asm__ __volatile__ ( \ |
198 | mb1 \ |
199 | " andnot %[__addr8],7,%[__addr64]\n" \ |
200 | " insbl %[__value],%[__addr8],%[__sval]\n" \ |
201 | "1: ldq_l %[__tmp],0(%[__addr64])\n" \ |
202 | " extbl %[__tmp],%[__addr8],%[__ret]\n" \ |
203 | " mskbl %[__tmp],%[__addr8],%[__tmp]\n" \ |
204 | " or %[__sval],%[__tmp],%[__tmp]\n" \ |
205 | " stq_c %[__tmp],0(%[__addr64])\n" \ |
206 | " beq %[__tmp],1b\n" \ |
207 | mb2 \ |
208 | : [__ret] "=&r" (__ret), \ |
209 | [__sval] "=&r" (__sval), \ |
210 | [__tmp] "=&r" (__tmp), \ |
211 | [__addr64] "=&r" (__addr64) \ |
212 | : [__addr8] "r" (mem), \ |
213 | [__value] "r" (value) \ |
214 | : "memory"); \ |
215 | __ret; }) |
216 | |
217 | #define __arch_exchange_16_int(mem, value, mb1, mb2) \ |
218 | ({ \ |
219 | unsigned long __tmp, __addr64, __sval; __typeof(*mem) __ret; \ |
220 | __asm__ __volatile__ ( \ |
221 | mb1 \ |
222 | " andnot %[__addr16],7,%[__addr64]\n" \ |
223 | " inswl %[__value],%[__addr16],%[__sval]\n" \ |
224 | "1: ldq_l %[__tmp],0(%[__addr64])\n" \ |
225 | " extwl %[__tmp],%[__addr16],%[__ret]\n" \ |
226 | " mskwl %[__tmp],%[__addr16],%[__tmp]\n" \ |
227 | " or %[__sval],%[__tmp],%[__tmp]\n" \ |
228 | " stq_c %[__tmp],0(%[__addr64])\n" \ |
229 | " beq %[__tmp],1b\n" \ |
230 | mb2 \ |
231 | : [__ret] "=&r" (__ret), \ |
232 | [__sval] "=&r" (__sval), \ |
233 | [__tmp] "=&r" (__tmp), \ |
234 | [__addr64] "=&r" (__addr64) \ |
235 | : [__addr16] "r" (mem), \ |
236 | [__value] "r" (value) \ |
237 | : "memory"); \ |
238 | __ret; }) |
239 | |
240 | #define __arch_exchange_32_int(mem, value, mb1, mb2) \ |
241 | ({ \ |
242 | signed int __tmp; __typeof(*mem) __ret; \ |
243 | __asm__ __volatile__ ( \ |
244 | mb1 \ |
245 | "1: ldl_l %[__ret],%[__mem]\n" \ |
246 | " mov %[__val],%[__tmp]\n" \ |
247 | " stl_c %[__tmp],%[__mem]\n" \ |
248 | " beq %[__tmp],1b\n" \ |
249 | mb2 \ |
250 | : [__ret] "=&r" (__ret), \ |
251 | [__tmp] "=&r" (__tmp) \ |
252 | : [__mem] "m" (*(mem)), \ |
253 | [__val] "Ir" (value) \ |
254 | : "memory"); \ |
255 | __ret; }) |
256 | |
257 | #define __arch_exchange_64_int(mem, value, mb1, mb2) \ |
258 | ({ \ |
259 | unsigned long __tmp; __typeof(*mem) __ret; \ |
260 | __asm__ __volatile__ ( \ |
261 | mb1 \ |
262 | "1: ldq_l %[__ret],%[__mem]\n" \ |
263 | " mov %[__val],%[__tmp]\n" \ |
264 | " stq_c %[__tmp],%[__mem]\n" \ |
265 | " beq %[__tmp],1b\n" \ |
266 | mb2 \ |
267 | : [__ret] "=&r" (__ret), \ |
268 | [__tmp] "=&r" (__tmp) \ |
269 | : [__mem] "m" (*(mem)), \ |
270 | [__val] "Ir" (value) \ |
271 | : "memory"); \ |
272 | __ret; }) |
273 | |
274 | #define atomic_exchange_acq(mem, value) \ |
275 | __atomic_val_bysize (__arch_exchange, int, mem, value, "", __MB) |
276 | |
277 | #define atomic_exchange_rel(mem, value) \ |
278 | __atomic_val_bysize (__arch_exchange, int, mem, value, __MB, "") |
279 | |
280 | |
281 | /* Atomically add value and return the previous (unincremented) value. */ |
282 | |
283 | #define __arch_exchange_and_add_8_int(mem, value, mb1, mb2) \ |
284 | ({ __builtin_trap (); 0; }) |
285 | |
286 | #define __arch_exchange_and_add_16_int(mem, value, mb1, mb2) \ |
287 | ({ __builtin_trap (); 0; }) |
288 | |
289 | #define __arch_exchange_and_add_32_int(mem, value, mb1, mb2) \ |
290 | ({ \ |
291 | signed int __tmp; __typeof(*mem) __ret; \ |
292 | __asm__ __volatile__ ( \ |
293 | mb1 \ |
294 | "1: ldl_l %[__ret],%[__mem]\n" \ |
295 | " addl %[__ret],%[__val],%[__tmp]\n" \ |
296 | " stl_c %[__tmp],%[__mem]\n" \ |
297 | " beq %[__tmp],1b\n" \ |
298 | mb2 \ |
299 | : [__ret] "=&r" (__ret), \ |
300 | [__tmp] "=&r" (__tmp) \ |
301 | : [__mem] "m" (*(mem)), \ |
302 | [__val] "Ir" ((signed int)(value)) \ |
303 | : "memory"); \ |
304 | __ret; }) |
305 | |
306 | #define __arch_exchange_and_add_64_int(mem, value, mb1, mb2) \ |
307 | ({ \ |
308 | unsigned long __tmp; __typeof(*mem) __ret; \ |
309 | __asm__ __volatile__ ( \ |
310 | mb1 \ |
311 | "1: ldq_l %[__ret],%[__mem]\n" \ |
312 | " addq %[__ret],%[__val],%[__tmp]\n" \ |
313 | " stq_c %[__tmp],%[__mem]\n" \ |
314 | " beq %[__tmp],1b\n" \ |
315 | mb2 \ |
316 | : [__ret] "=&r" (__ret), \ |
317 | [__tmp] "=&r" (__tmp) \ |
318 | : [__mem] "m" (*(mem)), \ |
319 | [__val] "Ir" ((unsigned long)(value)) \ |
320 | : "memory"); \ |
321 | __ret; }) |
322 | |
323 | /* ??? Barrier semantics for atomic_exchange_and_add appear to be |
324 | undefined. Use full barrier for now, as that's safe. */ |
325 | #define atomic_exchange_and_add(mem, value) \ |
326 | __atomic_val_bysize (__arch_exchange_and_add, int, mem, value, __MB, __MB) |
327 | |
328 | |
329 | /* ??? Blah, I'm lazy. Implement these later. Can do better than the |
330 | compare-and-exchange loop provided by generic code. |
331 | |
332 | #define atomic_decrement_if_positive(mem) |
333 | #define atomic_bit_test_set(mem, bit) |
334 | |
335 | */ |
336 | |
337 | #define atomic_full_barrier() __asm ("mb" : : : "memory"); |
338 | #define atomic_read_barrier() __asm ("mb" : : : "memory"); |
339 | #define atomic_write_barrier() __asm ("wmb" : : : "memory"); |
340 |
Warning: This file is not a C or C++ file. It does not have highlighting.