1 | #ifndef Py_ATOMIC_H |
2 | #define Py_ATOMIC_H |
3 | #ifdef Py_BUILD_CORE |
4 | |
5 | #include "dynamic_annotations.h" |
6 | |
7 | #include "pyconfig.h" |
8 | |
9 | #if defined(HAVE_STD_ATOMIC) |
10 | #include <stdatomic.h> |
11 | #endif |
12 | |
13 | /* This is modeled after the atomics interface from C1x, according to |
14 | * the draft at |
15 | * http://www.open-std.org/JTC1/SC22/wg14/www/docs/n1425.pdf. |
16 | * Operations and types are named the same except with a _Py_ prefix |
17 | * and have the same semantics. |
18 | * |
19 | * Beware, the implementations here are deep magic. |
20 | */ |
21 | |
22 | #if defined(HAVE_STD_ATOMIC) |
23 | |
24 | typedef enum _Py_memory_order { |
25 | _Py_memory_order_relaxed = memory_order_relaxed, |
26 | _Py_memory_order_acquire = memory_order_acquire, |
27 | _Py_memory_order_release = memory_order_release, |
28 | _Py_memory_order_acq_rel = memory_order_acq_rel, |
29 | _Py_memory_order_seq_cst = memory_order_seq_cst |
30 | } _Py_memory_order; |
31 | |
32 | typedef struct _Py_atomic_address { |
33 | atomic_uintptr_t _value; |
34 | } _Py_atomic_address; |
35 | |
36 | typedef struct _Py_atomic_int { |
37 | atomic_int _value; |
38 | } _Py_atomic_int; |
39 | |
40 | #define _Py_atomic_signal_fence(/*memory_order*/ ORDER) \ |
41 | atomic_signal_fence(ORDER) |
42 | |
43 | #define _Py_atomic_thread_fence(/*memory_order*/ ORDER) \ |
44 | atomic_thread_fence(ORDER) |
45 | |
46 | #define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ |
47 | atomic_store_explicit(&(ATOMIC_VAL)->_value, NEW_VAL, ORDER) |
48 | |
49 | #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ |
50 | atomic_load_explicit(&(ATOMIC_VAL)->_value, ORDER) |
51 | |
52 | /* Use builtin atomic operations in GCC >= 4.7 */ |
53 | #elif defined(HAVE_BUILTIN_ATOMIC) |
54 | |
55 | typedef enum _Py_memory_order { |
56 | _Py_memory_order_relaxed = __ATOMIC_RELAXED, |
57 | _Py_memory_order_acquire = __ATOMIC_ACQUIRE, |
58 | _Py_memory_order_release = __ATOMIC_RELEASE, |
59 | _Py_memory_order_acq_rel = __ATOMIC_ACQ_REL, |
60 | _Py_memory_order_seq_cst = __ATOMIC_SEQ_CST |
61 | } _Py_memory_order; |
62 | |
63 | typedef struct _Py_atomic_address { |
64 | Py_uintptr_t _value; |
65 | } _Py_atomic_address; |
66 | |
67 | typedef struct _Py_atomic_int { |
68 | int _value; |
69 | } _Py_atomic_int; |
70 | |
71 | #define _Py_atomic_signal_fence(/*memory_order*/ ORDER) \ |
72 | __atomic_signal_fence(ORDER) |
73 | |
74 | #define _Py_atomic_thread_fence(/*memory_order*/ ORDER) \ |
75 | __atomic_thread_fence(ORDER) |
76 | |
77 | #define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ |
78 | (assert((ORDER) == __ATOMIC_RELAXED \ |
79 | || (ORDER) == __ATOMIC_SEQ_CST \ |
80 | || (ORDER) == __ATOMIC_RELEASE), \ |
81 | __atomic_store_n(&(ATOMIC_VAL)->_value, NEW_VAL, ORDER)) |
82 | |
83 | #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ |
84 | (assert((ORDER) == __ATOMIC_RELAXED \ |
85 | || (ORDER) == __ATOMIC_SEQ_CST \ |
86 | || (ORDER) == __ATOMIC_ACQUIRE \ |
87 | || (ORDER) == __ATOMIC_CONSUME), \ |
88 | __atomic_load_n(&(ATOMIC_VAL)->_value, ORDER)) |
89 | |
90 | #else |
91 | |
92 | typedef enum _Py_memory_order { |
93 | _Py_memory_order_relaxed, |
94 | _Py_memory_order_acquire, |
95 | _Py_memory_order_release, |
96 | _Py_memory_order_acq_rel, |
97 | _Py_memory_order_seq_cst |
98 | } _Py_memory_order; |
99 | |
100 | typedef struct _Py_atomic_address { |
101 | Py_uintptr_t _value; |
102 | } _Py_atomic_address; |
103 | |
104 | typedef struct _Py_atomic_int { |
105 | int _value; |
106 | } _Py_atomic_int; |
107 | |
108 | /* Only support GCC (for expression statements) and x86 (for simple |
109 | * atomic semantics) for now */ |
110 | #if defined(__GNUC__) && (defined(__i386__) || defined(__amd64)) |
111 | |
112 | static __inline__ void |
113 | _Py_atomic_signal_fence(_Py_memory_order order) |
114 | { |
115 | if (order != _Py_memory_order_relaxed) |
116 | __asm__ volatile("" :::"memory" ); |
117 | } |
118 | |
119 | static __inline__ void |
120 | _Py_atomic_thread_fence(_Py_memory_order order) |
121 | { |
122 | if (order != _Py_memory_order_relaxed) |
123 | __asm__ volatile("mfence" :::"memory" ); |
124 | } |
125 | |
126 | /* Tell the race checker about this operation's effects. */ |
127 | static __inline__ void |
128 | _Py_ANNOTATE_MEMORY_ORDER(const volatile void *address, _Py_memory_order order) |
129 | { |
130 | (void)address; /* shut up -Wunused-parameter */ |
131 | switch(order) { |
132 | case _Py_memory_order_release: |
133 | case _Py_memory_order_acq_rel: |
134 | case _Py_memory_order_seq_cst: |
135 | _Py_ANNOTATE_HAPPENS_BEFORE(address); |
136 | break; |
137 | case _Py_memory_order_relaxed: |
138 | case _Py_memory_order_acquire: |
139 | break; |
140 | } |
141 | switch(order) { |
142 | case _Py_memory_order_acquire: |
143 | case _Py_memory_order_acq_rel: |
144 | case _Py_memory_order_seq_cst: |
145 | _Py_ANNOTATE_HAPPENS_AFTER(address); |
146 | break; |
147 | case _Py_memory_order_relaxed: |
148 | case _Py_memory_order_release: |
149 | break; |
150 | } |
151 | } |
152 | |
153 | #define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ |
154 | __extension__ ({ \ |
155 | __typeof__(ATOMIC_VAL) atomic_val = ATOMIC_VAL; \ |
156 | __typeof__(atomic_val->_value) new_val = NEW_VAL;\ |
157 | volatile __typeof__(new_val) *volatile_data = &atomic_val->_value; \ |
158 | _Py_memory_order order = ORDER; \ |
159 | _Py_ANNOTATE_MEMORY_ORDER(atomic_val, order); \ |
160 | \ |
161 | /* Perform the operation. */ \ |
162 | _Py_ANNOTATE_IGNORE_WRITES_BEGIN(); \ |
163 | switch(order) { \ |
164 | case _Py_memory_order_release: \ |
165 | _Py_atomic_signal_fence(_Py_memory_order_release); \ |
166 | /* fallthrough */ \ |
167 | case _Py_memory_order_relaxed: \ |
168 | *volatile_data = new_val; \ |
169 | break; \ |
170 | \ |
171 | case _Py_memory_order_acquire: \ |
172 | case _Py_memory_order_acq_rel: \ |
173 | case _Py_memory_order_seq_cst: \ |
174 | __asm__ volatile("xchg %0, %1" \ |
175 | : "+r"(new_val) \ |
176 | : "m"(atomic_val->_value) \ |
177 | : "memory"); \ |
178 | break; \ |
179 | } \ |
180 | _Py_ANNOTATE_IGNORE_WRITES_END(); \ |
181 | }) |
182 | |
183 | #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ |
184 | __extension__ ({ \ |
185 | __typeof__(ATOMIC_VAL) atomic_val = ATOMIC_VAL; \ |
186 | __typeof__(atomic_val->_value) result; \ |
187 | volatile __typeof__(result) *volatile_data = &atomic_val->_value; \ |
188 | _Py_memory_order order = ORDER; \ |
189 | _Py_ANNOTATE_MEMORY_ORDER(atomic_val, order); \ |
190 | \ |
191 | /* Perform the operation. */ \ |
192 | _Py_ANNOTATE_IGNORE_READS_BEGIN(); \ |
193 | switch(order) { \ |
194 | case _Py_memory_order_release: \ |
195 | case _Py_memory_order_acq_rel: \ |
196 | case _Py_memory_order_seq_cst: \ |
197 | /* Loads on x86 are not releases by default, so need a */ \ |
198 | /* thread fence. */ \ |
199 | _Py_atomic_thread_fence(_Py_memory_order_release); \ |
200 | break; \ |
201 | default: \ |
202 | /* No fence */ \ |
203 | break; \ |
204 | } \ |
205 | result = *volatile_data; \ |
206 | switch(order) { \ |
207 | case _Py_memory_order_acquire: \ |
208 | case _Py_memory_order_acq_rel: \ |
209 | case _Py_memory_order_seq_cst: \ |
210 | /* Loads on x86 are automatically acquire operations so */ \ |
211 | /* can get by with just a compiler fence. */ \ |
212 | _Py_atomic_signal_fence(_Py_memory_order_acquire); \ |
213 | break; \ |
214 | default: \ |
215 | /* No fence */ \ |
216 | break; \ |
217 | } \ |
218 | _Py_ANNOTATE_IGNORE_READS_END(); \ |
219 | result; \ |
220 | }) |
221 | |
222 | #else /* !gcc x86 */ |
223 | /* Fall back to other compilers and processors by assuming that simple |
224 | volatile accesses are atomic. This is false, so people should port |
225 | this. */ |
226 | #define _Py_atomic_signal_fence(/*memory_order*/ ORDER) ((void)0) |
227 | #define _Py_atomic_thread_fence(/*memory_order*/ ORDER) ((void)0) |
228 | #define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ |
229 | ((ATOMIC_VAL)->_value = NEW_VAL) |
230 | #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ |
231 | ((ATOMIC_VAL)->_value) |
232 | |
233 | #endif /* !gcc x86 */ |
234 | #endif |
235 | |
236 | /* Standardized shortcuts. */ |
237 | #define _Py_atomic_store(ATOMIC_VAL, NEW_VAL) \ |
238 | _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, _Py_memory_order_seq_cst) |
239 | #define _Py_atomic_load(ATOMIC_VAL) \ |
240 | _Py_atomic_load_explicit(ATOMIC_VAL, _Py_memory_order_seq_cst) |
241 | |
242 | /* Python-local extensions */ |
243 | |
244 | #define _Py_atomic_store_relaxed(ATOMIC_VAL, NEW_VAL) \ |
245 | _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, _Py_memory_order_relaxed) |
246 | #define _Py_atomic_load_relaxed(ATOMIC_VAL) \ |
247 | _Py_atomic_load_explicit(ATOMIC_VAL, _Py_memory_order_relaxed) |
248 | |
249 | #endif /* Py_BUILD_CORE */ |
250 | #endif /* Py_ATOMIC_H */ |
251 | |