1/*
2 * Generic barrier definitions.
3 *
4 * It should be possible to use these on really simple architectures,
5 * but it serves more as a starting point for new ports.
6 *
7 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
8 * Written by David Howells (dhowells@redhat.com)
9 *
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public Licence
12 * as published by the Free Software Foundation; either version
13 * 2 of the Licence, or (at your option) any later version.
14 */
15#ifndef __ASM_GENERIC_BARRIER_H
16#define __ASM_GENERIC_BARRIER_H
17
18#ifndef __ASSEMBLY__
19
20#include <linux/compiler.h>
21
22#ifndef nop
23#define nop() asm volatile ("nop")
24#endif
25
26/*
27 * Force strict CPU ordering. And yes, this is required on UP too when we're
28 * talking to devices.
29 *
30 * Fall back to compiler barriers if nothing better is provided.
31 */
32
33#ifndef mb
34#define mb() barrier()
35#endif
36
37#ifndef rmb
38#define rmb() mb()
39#endif
40
41#ifndef wmb
42#define wmb() mb()
43#endif
44
45#ifndef dma_rmb
46#define dma_rmb() rmb()
47#endif
48
49#ifndef dma_wmb
50#define dma_wmb() wmb()
51#endif
52
53#ifndef read_barrier_depends
54#define read_barrier_depends() do { } while (0)
55#endif
56
57#ifndef __smp_mb
58#define __smp_mb() mb()
59#endif
60
61#ifndef __smp_rmb
62#define __smp_rmb() rmb()
63#endif
64
65#ifndef __smp_wmb
66#define __smp_wmb() wmb()
67#endif
68
69#ifndef __smp_read_barrier_depends
70#define __smp_read_barrier_depends() read_barrier_depends()
71#endif
72
73#ifdef CONFIG_SMP
74
75#ifndef smp_mb
76#define smp_mb() __smp_mb()
77#endif
78
79#ifndef smp_rmb
80#define smp_rmb() __smp_rmb()
81#endif
82
83#ifndef smp_wmb
84#define smp_wmb() __smp_wmb()
85#endif
86
87#ifndef smp_read_barrier_depends
88#define smp_read_barrier_depends() __smp_read_barrier_depends()
89#endif
90
91#else /* !CONFIG_SMP */
92
93#ifndef smp_mb
94#define smp_mb() barrier()
95#endif
96
97#ifndef smp_rmb
98#define smp_rmb() barrier()
99#endif
100
101#ifndef smp_wmb
102#define smp_wmb() barrier()
103#endif
104
105#ifndef smp_read_barrier_depends
106#define smp_read_barrier_depends() do { } while (0)
107#endif
108
109#endif /* CONFIG_SMP */
110
111#ifndef __smp_store_mb
112#define __smp_store_mb(var, value) do { WRITE_ONCE(var, value); __smp_mb(); } while (0)
113#endif
114
115#ifndef __smp_mb__before_atomic
116#define __smp_mb__before_atomic() __smp_mb()
117#endif
118
119#ifndef __smp_mb__after_atomic
120#define __smp_mb__after_atomic() __smp_mb()
121#endif
122
123#ifndef __smp_store_release
124#define __smp_store_release(p, v) \
125do { \
126 compiletime_assert_atomic_type(*p); \
127 __smp_mb(); \
128 WRITE_ONCE(*p, v); \
129} while (0)
130#endif
131
132#ifndef __smp_load_acquire
133#define __smp_load_acquire(p) \
134({ \
135 typeof(*p) ___p1 = READ_ONCE(*p); \
136 compiletime_assert_atomic_type(*p); \
137 __smp_mb(); \
138 ___p1; \
139})
140#endif
141
142#ifdef CONFIG_SMP
143
144#ifndef smp_store_mb
145#define smp_store_mb(var, value) __smp_store_mb(var, value)
146#endif
147
148#ifndef smp_mb__before_atomic
149#define smp_mb__before_atomic() __smp_mb__before_atomic()
150#endif
151
152#ifndef smp_mb__after_atomic
153#define smp_mb__after_atomic() __smp_mb__after_atomic()
154#endif
155
156#ifndef smp_store_release
157#define smp_store_release(p, v) __smp_store_release(p, v)
158#endif
159
160#ifndef smp_load_acquire
161#define smp_load_acquire(p) __smp_load_acquire(p)
162#endif
163
164#else /* !CONFIG_SMP */
165
166#ifndef smp_store_mb
167#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); barrier(); } while (0)
168#endif
169
170#ifndef smp_mb__before_atomic
171#define smp_mb__before_atomic() barrier()
172#endif
173
174#ifndef smp_mb__after_atomic
175#define smp_mb__after_atomic() barrier()
176#endif
177
178#ifndef smp_store_release
179#define smp_store_release(p, v) \
180do { \
181 compiletime_assert_atomic_type(*p); \
182 barrier(); \
183 WRITE_ONCE(*p, v); \
184} while (0)
185#endif
186
187#ifndef smp_load_acquire
188#define smp_load_acquire(p) \
189({ \
190 typeof(*p) ___p1 = READ_ONCE(*p); \
191 compiletime_assert_atomic_type(*p); \
192 barrier(); \
193 ___p1; \
194})
195#endif
196
197#endif /* CONFIG_SMP */
198
199/* Barriers for virtual machine guests when talking to an SMP host */
200#define virt_mb() __smp_mb()
201#define virt_rmb() __smp_rmb()
202#define virt_wmb() __smp_wmb()
203#define virt_read_barrier_depends() __smp_read_barrier_depends()
204#define virt_store_mb(var, value) __smp_store_mb(var, value)
205#define virt_mb__before_atomic() __smp_mb__before_atomic()
206#define virt_mb__after_atomic() __smp_mb__after_atomic()
207#define virt_store_release(p, v) __smp_store_release(p, v)
208#define virt_load_acquire(p) __smp_load_acquire(p)
209
210/**
211 * smp_acquire__after_ctrl_dep() - Provide ACQUIRE ordering after a control dependency
212 *
213 * A control dependency provides a LOAD->STORE order, the additional RMB
214 * provides LOAD->LOAD order, together they provide LOAD->{LOAD,STORE} order,
215 * aka. (load)-ACQUIRE.
216 *
217 * Architectures that do not do load speculation can have this be barrier().
218 */
219#ifndef smp_acquire__after_ctrl_dep
220#define smp_acquire__after_ctrl_dep() smp_rmb()
221#endif
222
223/**
224 * smp_cond_load_relaxed() - (Spin) wait for cond with no ordering guarantees
225 * @ptr: pointer to the variable to wait on
226 * @cond: boolean expression to wait for
227 *
228 * Equivalent to using READ_ONCE() on the condition variable.
229 *
230 * Due to C lacking lambda expressions we load the value of *ptr into a
231 * pre-named variable @VAL to be used in @cond.
232 */
233#ifndef smp_cond_load_relaxed
234#define smp_cond_load_relaxed(ptr, cond_expr) ({ \
235 typeof(ptr) __PTR = (ptr); \
236 typeof(*ptr) VAL; \
237 for (;;) { \
238 VAL = READ_ONCE(*__PTR); \
239 if (cond_expr) \
240 break; \
241 cpu_relax(); \
242 } \
243 VAL; \
244})
245#endif
246
247/**
248 * smp_cond_load_acquire() - (Spin) wait for cond with ACQUIRE ordering
249 * @ptr: pointer to the variable to wait on
250 * @cond: boolean expression to wait for
251 *
252 * Equivalent to using smp_load_acquire() on the condition variable but employs
253 * the control dependency of the wait to reduce the barrier on many platforms.
254 */
255#ifndef smp_cond_load_acquire
256#define smp_cond_load_acquire(ptr, cond_expr) ({ \
257 typeof(*ptr) _val; \
258 _val = smp_cond_load_relaxed(ptr, cond_expr); \
259 smp_acquire__after_ctrl_dep(); \
260 _val; \
261})
262#endif
263
264#endif /* !__ASSEMBLY__ */
265#endif /* __ASM_GENERIC_BARRIER_H */
266