1#ifndef Py_ATOMIC_H
2#define Py_ATOMIC_H
3#ifdef Py_BUILD_CORE
4
5#include "dynamic_annotations.h"
6
7#include "pyconfig.h"
8
9#if defined(HAVE_STD_ATOMIC)
10#include <stdatomic.h>
11#endif
12
13/* This is modeled after the atomics interface from C1x, according to
14 * the draft at
15 * http://www.open-std.org/JTC1/SC22/wg14/www/docs/n1425.pdf.
16 * Operations and types are named the same except with a _Py_ prefix
17 * and have the same semantics.
18 *
19 * Beware, the implementations here are deep magic.
20 */
21
22#if defined(HAVE_STD_ATOMIC)
23
24typedef enum _Py_memory_order {
25 _Py_memory_order_relaxed = memory_order_relaxed,
26 _Py_memory_order_acquire = memory_order_acquire,
27 _Py_memory_order_release = memory_order_release,
28 _Py_memory_order_acq_rel = memory_order_acq_rel,
29 _Py_memory_order_seq_cst = memory_order_seq_cst
30} _Py_memory_order;
31
32typedef struct _Py_atomic_address {
33 atomic_uintptr_t _value;
34} _Py_atomic_address;
35
36typedef struct _Py_atomic_int {
37 atomic_int _value;
38} _Py_atomic_int;
39
40#define _Py_atomic_signal_fence(/*memory_order*/ ORDER) \
41 atomic_signal_fence(ORDER)
42
43#define _Py_atomic_thread_fence(/*memory_order*/ ORDER) \
44 atomic_thread_fence(ORDER)
45
46#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
47 atomic_store_explicit(&(ATOMIC_VAL)->_value, NEW_VAL, ORDER)
48
49#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \
50 atomic_load_explicit(&(ATOMIC_VAL)->_value, ORDER)
51
52/* Use builtin atomic operations in GCC >= 4.7 */
53#elif defined(HAVE_BUILTIN_ATOMIC)
54
55typedef enum _Py_memory_order {
56 _Py_memory_order_relaxed = __ATOMIC_RELAXED,
57 _Py_memory_order_acquire = __ATOMIC_ACQUIRE,
58 _Py_memory_order_release = __ATOMIC_RELEASE,
59 _Py_memory_order_acq_rel = __ATOMIC_ACQ_REL,
60 _Py_memory_order_seq_cst = __ATOMIC_SEQ_CST
61} _Py_memory_order;
62
63typedef struct _Py_atomic_address {
64 Py_uintptr_t _value;
65} _Py_atomic_address;
66
67typedef struct _Py_atomic_int {
68 int _value;
69} _Py_atomic_int;
70
71#define _Py_atomic_signal_fence(/*memory_order*/ ORDER) \
72 __atomic_signal_fence(ORDER)
73
74#define _Py_atomic_thread_fence(/*memory_order*/ ORDER) \
75 __atomic_thread_fence(ORDER)
76
77#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
78 (assert((ORDER) == __ATOMIC_RELAXED \
79 || (ORDER) == __ATOMIC_SEQ_CST \
80 || (ORDER) == __ATOMIC_RELEASE), \
81 __atomic_store_n(&(ATOMIC_VAL)->_value, NEW_VAL, ORDER))
82
83#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \
84 (assert((ORDER) == __ATOMIC_RELAXED \
85 || (ORDER) == __ATOMIC_SEQ_CST \
86 || (ORDER) == __ATOMIC_ACQUIRE \
87 || (ORDER) == __ATOMIC_CONSUME), \
88 __atomic_load_n(&(ATOMIC_VAL)->_value, ORDER))
89
90#else
91
92typedef enum _Py_memory_order {
93 _Py_memory_order_relaxed,
94 _Py_memory_order_acquire,
95 _Py_memory_order_release,
96 _Py_memory_order_acq_rel,
97 _Py_memory_order_seq_cst
98} _Py_memory_order;
99
100typedef struct _Py_atomic_address {
101 Py_uintptr_t _value;
102} _Py_atomic_address;
103
104typedef struct _Py_atomic_int {
105 int _value;
106} _Py_atomic_int;
107
108/* Only support GCC (for expression statements) and x86 (for simple
109 * atomic semantics) for now */
110#if defined(__GNUC__) && (defined(__i386__) || defined(__amd64))
111
112static __inline__ void
113_Py_atomic_signal_fence(_Py_memory_order order)
114{
115 if (order != _Py_memory_order_relaxed)
116 __asm__ volatile("":::"memory");
117}
118
119static __inline__ void
120_Py_atomic_thread_fence(_Py_memory_order order)
121{
122 if (order != _Py_memory_order_relaxed)
123 __asm__ volatile("mfence":::"memory");
124}
125
126/* Tell the race checker about this operation's effects. */
127static __inline__ void
128_Py_ANNOTATE_MEMORY_ORDER(const volatile void *address, _Py_memory_order order)
129{
130 (void)address; /* shut up -Wunused-parameter */
131 switch(order) {
132 case _Py_memory_order_release:
133 case _Py_memory_order_acq_rel:
134 case _Py_memory_order_seq_cst:
135 _Py_ANNOTATE_HAPPENS_BEFORE(address);
136 break;
137 case _Py_memory_order_relaxed:
138 case _Py_memory_order_acquire:
139 break;
140 }
141 switch(order) {
142 case _Py_memory_order_acquire:
143 case _Py_memory_order_acq_rel:
144 case _Py_memory_order_seq_cst:
145 _Py_ANNOTATE_HAPPENS_AFTER(address);
146 break;
147 case _Py_memory_order_relaxed:
148 case _Py_memory_order_release:
149 break;
150 }
151}
152
153#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
154 __extension__ ({ \
155 __typeof__(ATOMIC_VAL) atomic_val = ATOMIC_VAL; \
156 __typeof__(atomic_val->_value) new_val = NEW_VAL;\
157 volatile __typeof__(new_val) *volatile_data = &atomic_val->_value; \
158 _Py_memory_order order = ORDER; \
159 _Py_ANNOTATE_MEMORY_ORDER(atomic_val, order); \
160 \
161 /* Perform the operation. */ \
162 _Py_ANNOTATE_IGNORE_WRITES_BEGIN(); \
163 switch(order) { \
164 case _Py_memory_order_release: \
165 _Py_atomic_signal_fence(_Py_memory_order_release); \
166 /* fallthrough */ \
167 case _Py_memory_order_relaxed: \
168 *volatile_data = new_val; \
169 break; \
170 \
171 case _Py_memory_order_acquire: \
172 case _Py_memory_order_acq_rel: \
173 case _Py_memory_order_seq_cst: \
174 __asm__ volatile("xchg %0, %1" \
175 : "+r"(new_val) \
176 : "m"(atomic_val->_value) \
177 : "memory"); \
178 break; \
179 } \
180 _Py_ANNOTATE_IGNORE_WRITES_END(); \
181 })
182
183#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \
184 __extension__ ({ \
185 __typeof__(ATOMIC_VAL) atomic_val = ATOMIC_VAL; \
186 __typeof__(atomic_val->_value) result; \
187 volatile __typeof__(result) *volatile_data = &atomic_val->_value; \
188 _Py_memory_order order = ORDER; \
189 _Py_ANNOTATE_MEMORY_ORDER(atomic_val, order); \
190 \
191 /* Perform the operation. */ \
192 _Py_ANNOTATE_IGNORE_READS_BEGIN(); \
193 switch(order) { \
194 case _Py_memory_order_release: \
195 case _Py_memory_order_acq_rel: \
196 case _Py_memory_order_seq_cst: \
197 /* Loads on x86 are not releases by default, so need a */ \
198 /* thread fence. */ \
199 _Py_atomic_thread_fence(_Py_memory_order_release); \
200 break; \
201 default: \
202 /* No fence */ \
203 break; \
204 } \
205 result = *volatile_data; \
206 switch(order) { \
207 case _Py_memory_order_acquire: \
208 case _Py_memory_order_acq_rel: \
209 case _Py_memory_order_seq_cst: \
210 /* Loads on x86 are automatically acquire operations so */ \
211 /* can get by with just a compiler fence. */ \
212 _Py_atomic_signal_fence(_Py_memory_order_acquire); \
213 break; \
214 default: \
215 /* No fence */ \
216 break; \
217 } \
218 _Py_ANNOTATE_IGNORE_READS_END(); \
219 result; \
220 })
221
222#else /* !gcc x86 */
223/* Fall back to other compilers and processors by assuming that simple
224 volatile accesses are atomic. This is false, so people should port
225 this. */
226#define _Py_atomic_signal_fence(/*memory_order*/ ORDER) ((void)0)
227#define _Py_atomic_thread_fence(/*memory_order*/ ORDER) ((void)0)
228#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
229 ((ATOMIC_VAL)->_value = NEW_VAL)
230#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \
231 ((ATOMIC_VAL)->_value)
232
233#endif /* !gcc x86 */
234#endif
235
236/* Standardized shortcuts. */
237#define _Py_atomic_store(ATOMIC_VAL, NEW_VAL) \
238 _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, _Py_memory_order_seq_cst)
239#define _Py_atomic_load(ATOMIC_VAL) \
240 _Py_atomic_load_explicit(ATOMIC_VAL, _Py_memory_order_seq_cst)
241
242/* Python-local extensions */
243
244#define _Py_atomic_store_relaxed(ATOMIC_VAL, NEW_VAL) \
245 _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, _Py_memory_order_relaxed)
246#define _Py_atomic_load_relaxed(ATOMIC_VAL) \
247 _Py_atomic_load_explicit(ATOMIC_VAL, _Py_memory_order_relaxed)
248
249#endif /* Py_BUILD_CORE */
250#endif /* Py_ATOMIC_H */
251