1 | // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) |
2 | /* Copyright (C) 2015-2018 Netronome Systems, Inc. */ |
3 | |
4 | #include <linux/delay.h> |
5 | #include <linux/device.h> |
6 | #include <linux/jiffies.h> |
7 | #include <linux/types.h> |
8 | #include <linux/slab.h> |
9 | #include <linux/wait.h> |
10 | |
11 | #include "nfp_cpp.h" |
12 | #include "nfp6000/nfp6000.h" |
13 | |
14 | struct nfp_cpp_mutex { |
15 | struct nfp_cpp *cpp; |
16 | int target; |
17 | u16 depth; |
18 | unsigned long long address; |
19 | u32 key; |
20 | }; |
21 | |
22 | static u32 nfp_mutex_locked(u16 interface) |
23 | { |
24 | return (u32)interface << 16 | 0x000f; |
25 | } |
26 | |
27 | static u32 nfp_mutex_unlocked(u16 interface) |
28 | { |
29 | return (u32)interface << 16 | 0x0000; |
30 | } |
31 | |
32 | static u32 nfp_mutex_owner(u32 val) |
33 | { |
34 | return val >> 16; |
35 | } |
36 | |
37 | static bool nfp_mutex_is_locked(u32 val) |
38 | { |
39 | return (val & 0xffff) == 0x000f; |
40 | } |
41 | |
42 | static bool nfp_mutex_is_unlocked(u32 val) |
43 | { |
44 | return (val & 0xffff) == 0000; |
45 | } |
46 | |
47 | /* If you need more than 65536 recursive locks, please rethink your code. */ |
48 | #define NFP_MUTEX_DEPTH_MAX 0xffff |
49 | |
50 | static int |
51 | nfp_cpp_mutex_validate(u16 interface, int *target, unsigned long long address) |
52 | { |
53 | /* Not permitted on invalid interfaces */ |
54 | if (NFP_CPP_INTERFACE_TYPE_of(interface) == |
55 | NFP_CPP_INTERFACE_TYPE_INVALID) |
56 | return -EINVAL; |
57 | |
58 | /* Address must be 64-bit aligned */ |
59 | if (address & 7) |
60 | return -EINVAL; |
61 | |
62 | if (*target != NFP_CPP_TARGET_MU) |
63 | return -EINVAL; |
64 | |
65 | return 0; |
66 | } |
67 | |
68 | /** |
69 | * nfp_cpp_mutex_init() - Initialize a mutex location |
70 | * @cpp: NFP CPP handle |
71 | * @target: NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU) |
72 | * @address: Offset into the address space of the NFP CPP target ID |
73 | * @key: Unique 32-bit value for this mutex |
74 | * |
75 | * The CPP target:address must point to a 64-bit aligned location, and |
76 | * will initialize 64 bits of data at the location. |
77 | * |
78 | * This creates the initial mutex state, as locked by this |
79 | * nfp_cpp_interface(). |
80 | * |
81 | * This function should only be called when setting up |
82 | * the initial lock state upon boot-up of the system. |
83 | * |
84 | * Return: 0 on success, or -errno on failure |
85 | */ |
86 | int nfp_cpp_mutex_init(struct nfp_cpp *cpp, |
87 | int target, unsigned long long address, u32 key) |
88 | { |
89 | const u32 muw = NFP_CPP_ID(target, 4, 0); /* atomic_write */ |
90 | u16 interface = nfp_cpp_interface(cpp); |
91 | int err; |
92 | |
93 | err = nfp_cpp_mutex_validate(interface, target: &target, address); |
94 | if (err) |
95 | return err; |
96 | |
97 | err = nfp_cpp_writel(cpp, cpp_id: muw, address: address + 4, value: key); |
98 | if (err) |
99 | return err; |
100 | |
101 | err = nfp_cpp_writel(cpp, cpp_id: muw, address, value: nfp_mutex_locked(interface)); |
102 | if (err) |
103 | return err; |
104 | |
105 | return 0; |
106 | } |
107 | |
108 | /** |
109 | * nfp_cpp_mutex_alloc() - Create a mutex handle |
110 | * @cpp: NFP CPP handle |
111 | * @target: NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU) |
112 | * @address: Offset into the address space of the NFP CPP target ID |
113 | * @key: 32-bit unique key (must match the key at this location) |
114 | * |
115 | * The CPP target:address must point to a 64-bit aligned location, and |
116 | * reserve 64 bits of data at the location for use by the handle. |
117 | * |
118 | * Only target/address pairs that point to entities that support the |
119 | * MU Atomic Engine's CmpAndSwap32 command are supported. |
120 | * |
121 | * Return: A non-NULL struct nfp_cpp_mutex * on success, NULL on failure. |
122 | */ |
123 | struct nfp_cpp_mutex *nfp_cpp_mutex_alloc(struct nfp_cpp *cpp, int target, |
124 | unsigned long long address, u32 key) |
125 | { |
126 | const u32 mur = NFP_CPP_ID(target, 3, 0); /* atomic_read */ |
127 | u16 interface = nfp_cpp_interface(cpp); |
128 | struct nfp_cpp_mutex *mutex; |
129 | int err; |
130 | u32 tmp; |
131 | |
132 | err = nfp_cpp_mutex_validate(interface, target: &target, address); |
133 | if (err) |
134 | return NULL; |
135 | |
136 | err = nfp_cpp_readl(cpp, cpp_id: mur, address: address + 4, value: &tmp); |
137 | if (err < 0) |
138 | return NULL; |
139 | |
140 | if (tmp != key) |
141 | return NULL; |
142 | |
143 | mutex = kzalloc(size: sizeof(*mutex), GFP_KERNEL); |
144 | if (!mutex) |
145 | return NULL; |
146 | |
147 | mutex->cpp = cpp; |
148 | mutex->target = target; |
149 | mutex->address = address; |
150 | mutex->key = key; |
151 | mutex->depth = 0; |
152 | |
153 | return mutex; |
154 | } |
155 | |
156 | /** |
157 | * nfp_cpp_mutex_free() - Free a mutex handle - does not alter the lock state |
158 | * @mutex: NFP CPP Mutex handle |
159 | */ |
160 | void nfp_cpp_mutex_free(struct nfp_cpp_mutex *mutex) |
161 | { |
162 | kfree(objp: mutex); |
163 | } |
164 | |
165 | /** |
166 | * nfp_cpp_mutex_lock() - Lock a mutex handle, using the NFP MU Atomic Engine |
167 | * @mutex: NFP CPP Mutex handle |
168 | * |
169 | * Return: 0 on success, or -errno on failure |
170 | */ |
171 | int nfp_cpp_mutex_lock(struct nfp_cpp_mutex *mutex) |
172 | { |
173 | unsigned long warn_at = jiffies + NFP_MUTEX_WAIT_FIRST_WARN * HZ; |
174 | unsigned long err_at = jiffies + NFP_MUTEX_WAIT_ERROR * HZ; |
175 | unsigned int timeout_ms = 1; |
176 | int err; |
177 | |
178 | /* We can't use a waitqueue here, because the unlocker |
179 | * might be on a separate CPU. |
180 | * |
181 | * So just wait for now. |
182 | */ |
183 | for (;;) { |
184 | err = nfp_cpp_mutex_trylock(mutex); |
185 | if (err != -EBUSY) |
186 | break; |
187 | |
188 | err = msleep_interruptible(msecs: timeout_ms); |
189 | if (err != 0) { |
190 | nfp_info(mutex->cpp, |
191 | "interrupted waiting for NFP mutex\n" ); |
192 | return -ERESTARTSYS; |
193 | } |
194 | |
195 | if (time_is_before_eq_jiffies(warn_at)) { |
196 | warn_at = jiffies + NFP_MUTEX_WAIT_NEXT_WARN * HZ; |
197 | nfp_warn(mutex->cpp, |
198 | "Warning: waiting for NFP mutex [depth:%hd target:%d addr:%llx key:%08x]\n" , |
199 | mutex->depth, |
200 | mutex->target, mutex->address, mutex->key); |
201 | } |
202 | if (time_is_before_eq_jiffies(err_at)) { |
203 | nfp_err(mutex->cpp, "Error: mutex wait timed out\n" ); |
204 | return -EBUSY; |
205 | } |
206 | } |
207 | |
208 | return err; |
209 | } |
210 | |
211 | /** |
212 | * nfp_cpp_mutex_unlock() - Unlock a mutex handle, using the MU Atomic Engine |
213 | * @mutex: NFP CPP Mutex handle |
214 | * |
215 | * Return: 0 on success, or -errno on failure |
216 | */ |
217 | int nfp_cpp_mutex_unlock(struct nfp_cpp_mutex *mutex) |
218 | { |
219 | const u32 muw = NFP_CPP_ID(mutex->target, 4, 0); /* atomic_write */ |
220 | const u32 mur = NFP_CPP_ID(mutex->target, 3, 0); /* atomic_read */ |
221 | struct nfp_cpp *cpp = mutex->cpp; |
222 | u32 key, value; |
223 | u16 interface; |
224 | int err; |
225 | |
226 | interface = nfp_cpp_interface(cpp); |
227 | |
228 | if (mutex->depth > 1) { |
229 | mutex->depth--; |
230 | return 0; |
231 | } |
232 | |
233 | err = nfp_cpp_readl(cpp: mutex->cpp, cpp_id: mur, address: mutex->address + 4, value: &key); |
234 | if (err < 0) |
235 | return err; |
236 | |
237 | if (key != mutex->key) |
238 | return -EPERM; |
239 | |
240 | err = nfp_cpp_readl(cpp: mutex->cpp, cpp_id: mur, address: mutex->address, value: &value); |
241 | if (err < 0) |
242 | return err; |
243 | |
244 | if (value != nfp_mutex_locked(interface)) |
245 | return -EACCES; |
246 | |
247 | err = nfp_cpp_writel(cpp, cpp_id: muw, address: mutex->address, |
248 | value: nfp_mutex_unlocked(interface)); |
249 | if (err < 0) |
250 | return err; |
251 | |
252 | mutex->depth = 0; |
253 | return 0; |
254 | } |
255 | |
256 | /** |
257 | * nfp_cpp_mutex_trylock() - Attempt to lock a mutex handle |
258 | * @mutex: NFP CPP Mutex handle |
259 | * |
260 | * Return: 0 if the lock succeeded, -errno on failure |
261 | */ |
262 | int nfp_cpp_mutex_trylock(struct nfp_cpp_mutex *mutex) |
263 | { |
264 | const u32 muw = NFP_CPP_ID(mutex->target, 4, 0); /* atomic_write */ |
265 | const u32 mus = NFP_CPP_ID(mutex->target, 5, 3); /* test_set_imm */ |
266 | const u32 mur = NFP_CPP_ID(mutex->target, 3, 0); /* atomic_read */ |
267 | struct nfp_cpp *cpp = mutex->cpp; |
268 | u32 key, value, tmp; |
269 | int err; |
270 | |
271 | if (mutex->depth > 0) { |
272 | if (mutex->depth == NFP_MUTEX_DEPTH_MAX) |
273 | return -E2BIG; |
274 | mutex->depth++; |
275 | return 0; |
276 | } |
277 | |
278 | /* Verify that the lock marker is not damaged */ |
279 | err = nfp_cpp_readl(cpp, cpp_id: mur, address: mutex->address + 4, value: &key); |
280 | if (err < 0) |
281 | return err; |
282 | |
283 | if (key != mutex->key) |
284 | return -EPERM; |
285 | |
286 | /* Compare against the unlocked state, and if true, |
287 | * write the interface id into the top 16 bits, and |
288 | * mark as locked. |
289 | */ |
290 | value = nfp_mutex_locked(interface: nfp_cpp_interface(cpp)); |
291 | |
292 | /* We use test_set_imm here, as it implies a read |
293 | * of the current state, and sets the bits in the |
294 | * bytemask of the command to 1s. Since the mutex |
295 | * is guaranteed to be 64-bit aligned, the bytemask |
296 | * of this 32-bit command is ensured to be 8'b00001111, |
297 | * which implies that the lower 4 bits will be set to |
298 | * ones regardless of the initial state. |
299 | * |
300 | * Since this is a 'Readback' operation, with no Pull |
301 | * data, we can treat this as a normal Push (read) |
302 | * atomic, which returns the original value. |
303 | */ |
304 | err = nfp_cpp_readl(cpp, cpp_id: mus, address: mutex->address, value: &tmp); |
305 | if (err < 0) |
306 | return err; |
307 | |
308 | /* Was it unlocked? */ |
309 | if (nfp_mutex_is_unlocked(val: tmp)) { |
310 | /* The read value can only be 0x....0000 in the unlocked state. |
311 | * If there was another contending for this lock, then |
312 | * the lock state would be 0x....000f |
313 | */ |
314 | |
315 | /* Write our owner ID into the lock |
316 | * While not strictly necessary, this helps with |
317 | * debug and bookkeeping. |
318 | */ |
319 | err = nfp_cpp_writel(cpp, cpp_id: muw, address: mutex->address, value); |
320 | if (err < 0) |
321 | return err; |
322 | |
323 | mutex->depth = 1; |
324 | return 0; |
325 | } |
326 | |
327 | return nfp_mutex_is_locked(val: tmp) ? -EBUSY : -EINVAL; |
328 | } |
329 | |
330 | /** |
331 | * nfp_cpp_mutex_reclaim() - Unlock mutex if held by local endpoint |
332 | * @cpp: NFP CPP handle |
333 | * @target: NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU) |
334 | * @address: Offset into the address space of the NFP CPP target ID |
335 | * |
336 | * Release lock if held by local system. Extreme care is advised, call only |
337 | * when no local lock users can exist. |
338 | * |
339 | * Return: 0 if the lock was OK, 1 if locked by us, -errno on invalid mutex |
340 | */ |
341 | int nfp_cpp_mutex_reclaim(struct nfp_cpp *cpp, int target, |
342 | unsigned long long address) |
343 | { |
344 | const u32 mur = NFP_CPP_ID(target, 3, 0); /* atomic_read */ |
345 | const u32 muw = NFP_CPP_ID(target, 4, 0); /* atomic_write */ |
346 | u16 interface = nfp_cpp_interface(cpp); |
347 | int err; |
348 | u32 tmp; |
349 | |
350 | err = nfp_cpp_mutex_validate(interface, target: &target, address); |
351 | if (err) |
352 | return err; |
353 | |
354 | /* Check lock */ |
355 | err = nfp_cpp_readl(cpp, cpp_id: mur, address, value: &tmp); |
356 | if (err < 0) |
357 | return err; |
358 | |
359 | if (nfp_mutex_is_unlocked(val: tmp) || nfp_mutex_owner(val: tmp) != interface) |
360 | return 0; |
361 | |
362 | /* Bust the lock */ |
363 | err = nfp_cpp_writel(cpp, cpp_id: muw, address, value: nfp_mutex_unlocked(interface)); |
364 | if (err < 0) |
365 | return err; |
366 | |
367 | return 1; |
368 | } |
369 | |