1 | // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) |
2 | /* Copyright (C) 2019 Netronome Systems, Inc. */ |
3 | |
4 | #include <linux/bitfield.h> |
5 | #include <linux/io.h> |
6 | #include <linux/skbuff.h> |
7 | |
8 | #include "ccm.h" |
9 | #include "nfp_net.h" |
10 | |
11 | /* CCM messages via the mailbox. CMSGs get wrapped into simple TLVs |
12 | * and copied into the mailbox. Multiple messages can be copied to |
13 | * form a batch. Threads come in with CMSG formed in an skb, then |
14 | * enqueue that skb onto the request queue. If threads skb is first |
15 | * in queue this thread will handle the mailbox operation. It copies |
16 | * up to 64 messages into the mailbox (making sure that both requests |
17 | * and replies will fit. After FW is done processing the batch it |
18 | * copies the data out and wakes waiting threads. |
19 | * If a thread is waiting it either gets its the message completed |
20 | * (response is copied into the same skb as the request, overwriting |
21 | * it), or becomes the first in queue. |
22 | * Completions and next-to-run are signaled via the control buffer |
23 | * to limit potential cache line bounces. |
24 | */ |
25 | |
26 | #define NFP_CCM_MBOX_BATCH_LIMIT 64 |
27 | #define NFP_CCM_TIMEOUT (NFP_NET_POLL_TIMEOUT * 1000) |
28 | #define NFP_CCM_MAX_QLEN 1024 |
29 | |
30 | enum nfp_net_mbox_cmsg_state { |
31 | NFP_NET_MBOX_CMSG_STATE_QUEUED, |
32 | NFP_NET_MBOX_CMSG_STATE_NEXT, |
33 | NFP_NET_MBOX_CMSG_STATE_BUSY, |
34 | NFP_NET_MBOX_CMSG_STATE_REPLY_FOUND, |
35 | NFP_NET_MBOX_CMSG_STATE_DONE, |
36 | }; |
37 | |
38 | /** |
39 | * struct nfp_ccm_mbox_cmsg_cb - CCM mailbox specific info |
40 | * @state: processing state (/stage) of the message |
41 | * @err: error encountered during processing if any |
42 | * @max_len: max(request_len, reply_len) |
43 | * @exp_reply: expected reply length (0 means don't validate) |
44 | * @posted: the message was posted and nobody waits for the reply |
45 | */ |
46 | struct nfp_ccm_mbox_cmsg_cb { |
47 | enum nfp_net_mbox_cmsg_state state; |
48 | int err; |
49 | unsigned int max_len; |
50 | unsigned int exp_reply; |
51 | bool posted; |
52 | }; |
53 | |
54 | static u32 nfp_ccm_mbox_max_msg(struct nfp_net *nn) |
55 | { |
56 | return round_down(nn->tlv_caps.mbox_len, 4) - |
57 | NFP_NET_CFG_MBOX_SIMPLE_VAL - /* common mbox command header */ |
58 | 4 * 2; /* Msg TLV plus End TLV headers */ |
59 | } |
60 | |
61 | static void |
62 | nfp_ccm_mbox_msg_init(struct sk_buff *skb, unsigned int exp_reply, int max_len) |
63 | { |
64 | struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb; |
65 | |
66 | cb->state = NFP_NET_MBOX_CMSG_STATE_QUEUED; |
67 | cb->err = 0; |
68 | cb->max_len = max_len; |
69 | cb->exp_reply = exp_reply; |
70 | cb->posted = false; |
71 | } |
72 | |
73 | static int nfp_ccm_mbox_maxlen(const struct sk_buff *skb) |
74 | { |
75 | struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb; |
76 | |
77 | return cb->max_len; |
78 | } |
79 | |
80 | static bool nfp_ccm_mbox_done(struct sk_buff *skb) |
81 | { |
82 | struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb; |
83 | |
84 | return cb->state == NFP_NET_MBOX_CMSG_STATE_DONE; |
85 | } |
86 | |
87 | static bool nfp_ccm_mbox_in_progress(struct sk_buff *skb) |
88 | { |
89 | struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb; |
90 | |
91 | return cb->state != NFP_NET_MBOX_CMSG_STATE_QUEUED && |
92 | cb->state != NFP_NET_MBOX_CMSG_STATE_NEXT; |
93 | } |
94 | |
95 | static void nfp_ccm_mbox_set_busy(struct sk_buff *skb) |
96 | { |
97 | struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb; |
98 | |
99 | cb->state = NFP_NET_MBOX_CMSG_STATE_BUSY; |
100 | } |
101 | |
102 | static bool nfp_ccm_mbox_is_posted(struct sk_buff *skb) |
103 | { |
104 | struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb; |
105 | |
106 | return cb->posted; |
107 | } |
108 | |
109 | static void nfp_ccm_mbox_mark_posted(struct sk_buff *skb) |
110 | { |
111 | struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb; |
112 | |
113 | cb->posted = true; |
114 | } |
115 | |
116 | static bool nfp_ccm_mbox_is_first(struct nfp_net *nn, struct sk_buff *skb) |
117 | { |
118 | return skb_queue_is_first(list: &nn->mbox_cmsg.queue, skb); |
119 | } |
120 | |
121 | static bool nfp_ccm_mbox_should_run(struct nfp_net *nn, struct sk_buff *skb) |
122 | { |
123 | struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb; |
124 | |
125 | return cb->state == NFP_NET_MBOX_CMSG_STATE_NEXT; |
126 | } |
127 | |
128 | static void nfp_ccm_mbox_mark_next_runner(struct nfp_net *nn) |
129 | { |
130 | struct nfp_ccm_mbox_cmsg_cb *cb; |
131 | struct sk_buff *skb; |
132 | |
133 | skb = skb_peek(list_: &nn->mbox_cmsg.queue); |
134 | if (!skb) |
135 | return; |
136 | |
137 | cb = (void *)skb->cb; |
138 | cb->state = NFP_NET_MBOX_CMSG_STATE_NEXT; |
139 | if (cb->posted) |
140 | queue_work(wq: nn->mbox_cmsg.workq, work: &nn->mbox_cmsg.runq_work); |
141 | } |
142 | |
143 | static void |
144 | nfp_ccm_mbox_write_tlv(struct nfp_net *nn, u32 off, u32 type, u32 len) |
145 | { |
146 | nn_writel(nn, off, |
147 | FIELD_PREP(NFP_NET_MBOX_TLV_TYPE, type) | |
148 | FIELD_PREP(NFP_NET_MBOX_TLV_LEN, len)); |
149 | } |
150 | |
151 | static void nfp_ccm_mbox_copy_in(struct nfp_net *nn, struct sk_buff *last) |
152 | { |
153 | struct sk_buff *skb; |
154 | int reserve, i, cnt; |
155 | __be32 *data; |
156 | u32 off, len; |
157 | |
158 | off = nn->tlv_caps.mbox_off + NFP_NET_CFG_MBOX_SIMPLE_VAL; |
159 | skb = __skb_peek(list_: &nn->mbox_cmsg.queue); |
160 | while (true) { |
161 | nfp_ccm_mbox_write_tlv(nn, off, type: NFP_NET_MBOX_TLV_TYPE_MSG, |
162 | len: skb->len); |
163 | off += 4; |
164 | |
165 | /* Write data word by word, skb->data should be aligned */ |
166 | data = (__be32 *)skb->data; |
167 | cnt = skb->len / 4; |
168 | for (i = 0 ; i < cnt; i++) { |
169 | nn_writel(nn, off, be32_to_cpu(data[i])); |
170 | off += 4; |
171 | } |
172 | if (skb->len & 3) { |
173 | __be32 tmp = 0; |
174 | |
175 | memcpy(&tmp, &data[i], skb->len & 3); |
176 | nn_writel(nn, off, be32_to_cpu(tmp)); |
177 | off += 4; |
178 | } |
179 | |
180 | /* Reserve space if reply is bigger */ |
181 | len = round_up(skb->len, 4); |
182 | reserve = nfp_ccm_mbox_maxlen(skb) - len; |
183 | if (reserve > 0) { |
184 | nfp_ccm_mbox_write_tlv(nn, off, |
185 | type: NFP_NET_MBOX_TLV_TYPE_RESV, |
186 | len: reserve); |
187 | off += 4 + reserve; |
188 | } |
189 | |
190 | if (skb == last) |
191 | break; |
192 | skb = skb_queue_next(list: &nn->mbox_cmsg.queue, skb); |
193 | } |
194 | |
195 | nfp_ccm_mbox_write_tlv(nn, off, type: NFP_NET_MBOX_TLV_TYPE_END, len: 0); |
196 | } |
197 | |
198 | static struct sk_buff * |
199 | nfp_ccm_mbox_find_req(struct nfp_net *nn, __be16 tag, struct sk_buff *last) |
200 | { |
201 | struct sk_buff *skb; |
202 | |
203 | skb = __skb_peek(list_: &nn->mbox_cmsg.queue); |
204 | while (true) { |
205 | if (__nfp_ccm_get_tag(skb) == tag) |
206 | return skb; |
207 | |
208 | if (skb == last) |
209 | return NULL; |
210 | skb = skb_queue_next(list: &nn->mbox_cmsg.queue, skb); |
211 | } |
212 | } |
213 | |
214 | static void nfp_ccm_mbox_copy_out(struct nfp_net *nn, struct sk_buff *last) |
215 | { |
216 | struct nfp_ccm_mbox_cmsg_cb *cb; |
217 | u8 __iomem *data, *end; |
218 | struct sk_buff *skb; |
219 | |
220 | data = nn->dp.ctrl_bar + nn->tlv_caps.mbox_off + |
221 | NFP_NET_CFG_MBOX_SIMPLE_VAL; |
222 | end = data + nn->tlv_caps.mbox_len; |
223 | |
224 | while (true) { |
225 | unsigned int length, offset, type; |
226 | struct nfp_ccm_hdr hdr; |
227 | u32 tlv_hdr; |
228 | |
229 | tlv_hdr = readl(addr: data); |
230 | type = FIELD_GET(NFP_NET_MBOX_TLV_TYPE, tlv_hdr); |
231 | length = FIELD_GET(NFP_NET_MBOX_TLV_LEN, tlv_hdr); |
232 | offset = data - nn->dp.ctrl_bar; |
233 | |
234 | /* Advance past the header */ |
235 | data += 4; |
236 | |
237 | if (data + length > end) { |
238 | nn_dp_warn(&nn->dp, "mailbox oversized TLV type:%d offset:%u len:%u\n" , |
239 | type, offset, length); |
240 | break; |
241 | } |
242 | |
243 | if (type == NFP_NET_MBOX_TLV_TYPE_END) |
244 | break; |
245 | if (type == NFP_NET_MBOX_TLV_TYPE_RESV) |
246 | goto next_tlv; |
247 | if (type != NFP_NET_MBOX_TLV_TYPE_MSG && |
248 | type != NFP_NET_MBOX_TLV_TYPE_MSG_NOSUP) { |
249 | nn_dp_warn(&nn->dp, "mailbox unknown TLV type:%d offset:%u len:%u\n" , |
250 | type, offset, length); |
251 | break; |
252 | } |
253 | |
254 | if (length < 4) { |
255 | nn_dp_warn(&nn->dp, "mailbox msg too short to contain header TLV type:%d offset:%u len:%u\n" , |
256 | type, offset, length); |
257 | break; |
258 | } |
259 | |
260 | hdr.raw = cpu_to_be32(readl(data)); |
261 | |
262 | skb = nfp_ccm_mbox_find_req(nn, tag: hdr.tag, last); |
263 | if (!skb) { |
264 | nn_dp_warn(&nn->dp, "mailbox request not found:%u\n" , |
265 | be16_to_cpu(hdr.tag)); |
266 | break; |
267 | } |
268 | cb = (void *)skb->cb; |
269 | |
270 | if (type == NFP_NET_MBOX_TLV_TYPE_MSG_NOSUP) { |
271 | nn_dp_warn(&nn->dp, |
272 | "mailbox msg not supported type:%d\n" , |
273 | nfp_ccm_get_type(skb)); |
274 | cb->err = -EIO; |
275 | goto next_tlv; |
276 | } |
277 | |
278 | if (hdr.type != __NFP_CCM_REPLY(nfp_ccm_get_type(skb))) { |
279 | nn_dp_warn(&nn->dp, "mailbox msg reply wrong type:%u expected:%lu\n" , |
280 | hdr.type, |
281 | __NFP_CCM_REPLY(nfp_ccm_get_type(skb))); |
282 | cb->err = -EIO; |
283 | goto next_tlv; |
284 | } |
285 | if (cb->exp_reply && length != cb->exp_reply) { |
286 | nn_dp_warn(&nn->dp, "mailbox msg reply wrong size type:%u expected:%u have:%u\n" , |
287 | hdr.type, length, cb->exp_reply); |
288 | cb->err = -EIO; |
289 | goto next_tlv; |
290 | } |
291 | if (length > cb->max_len) { |
292 | nn_dp_warn(&nn->dp, "mailbox msg oversized reply type:%u max:%u have:%u\n" , |
293 | hdr.type, cb->max_len, length); |
294 | cb->err = -EIO; |
295 | goto next_tlv; |
296 | } |
297 | |
298 | if (!cb->posted) { |
299 | __be32 *skb_data; |
300 | int i, cnt; |
301 | |
302 | if (length <= skb->len) |
303 | __skb_trim(skb, len: length); |
304 | else |
305 | skb_put(skb, len: length - skb->len); |
306 | |
307 | /* We overcopy here slightly, but that's okay, |
308 | * the skb is large enough, and the garbage will |
309 | * be ignored (beyond skb->len). |
310 | */ |
311 | skb_data = (__be32 *)skb->data; |
312 | memcpy(skb_data, &hdr, 4); |
313 | |
314 | cnt = DIV_ROUND_UP(length, 4); |
315 | for (i = 1 ; i < cnt; i++) |
316 | skb_data[i] = cpu_to_be32(readl(data + i * 4)); |
317 | } |
318 | |
319 | cb->state = NFP_NET_MBOX_CMSG_STATE_REPLY_FOUND; |
320 | next_tlv: |
321 | data += round_up(length, 4); |
322 | if (data + 4 > end) { |
323 | nn_dp_warn(&nn->dp, |
324 | "reached end of MBOX without END TLV\n" ); |
325 | break; |
326 | } |
327 | } |
328 | |
329 | smp_wmb(); /* order the skb->data vs. cb->state */ |
330 | spin_lock_bh(lock: &nn->mbox_cmsg.queue.lock); |
331 | do { |
332 | skb = __skb_dequeue(list: &nn->mbox_cmsg.queue); |
333 | cb = (void *)skb->cb; |
334 | |
335 | if (cb->state != NFP_NET_MBOX_CMSG_STATE_REPLY_FOUND) { |
336 | cb->err = -ENOENT; |
337 | smp_wmb(); /* order the cb->err vs. cb->state */ |
338 | } |
339 | cb->state = NFP_NET_MBOX_CMSG_STATE_DONE; |
340 | |
341 | if (cb->posted) { |
342 | if (cb->err) |
343 | nn_dp_warn(&nn->dp, |
344 | "mailbox posted msg failed type:%u err:%d\n" , |
345 | nfp_ccm_get_type(skb), cb->err); |
346 | dev_consume_skb_any(skb); |
347 | } |
348 | } while (skb != last); |
349 | |
350 | nfp_ccm_mbox_mark_next_runner(nn); |
351 | spin_unlock_bh(lock: &nn->mbox_cmsg.queue.lock); |
352 | } |
353 | |
354 | static void |
355 | nfp_ccm_mbox_mark_all_err(struct nfp_net *nn, struct sk_buff *last, int err) |
356 | { |
357 | struct nfp_ccm_mbox_cmsg_cb *cb; |
358 | struct sk_buff *skb; |
359 | |
360 | spin_lock_bh(lock: &nn->mbox_cmsg.queue.lock); |
361 | do { |
362 | skb = __skb_dequeue(list: &nn->mbox_cmsg.queue); |
363 | cb = (void *)skb->cb; |
364 | |
365 | cb->err = err; |
366 | smp_wmb(); /* order the cb->err vs. cb->state */ |
367 | cb->state = NFP_NET_MBOX_CMSG_STATE_DONE; |
368 | } while (skb != last); |
369 | |
370 | nfp_ccm_mbox_mark_next_runner(nn); |
371 | spin_unlock_bh(lock: &nn->mbox_cmsg.queue.lock); |
372 | } |
373 | |
374 | static void nfp_ccm_mbox_run_queue_unlock(struct nfp_net *nn) |
375 | __releases(&nn->mbox_cmsg.queue.lock) |
376 | { |
377 | int space = nn->tlv_caps.mbox_len - NFP_NET_CFG_MBOX_SIMPLE_VAL; |
378 | struct sk_buff *skb, *last; |
379 | int cnt, err; |
380 | |
381 | space -= 4; /* for End TLV */ |
382 | |
383 | /* First skb must fit, because it's ours and we checked it fits */ |
384 | cnt = 1; |
385 | last = skb = __skb_peek(list_: &nn->mbox_cmsg.queue); |
386 | space -= 4 + nfp_ccm_mbox_maxlen(skb); |
387 | |
388 | while (!skb_queue_is_last(list: &nn->mbox_cmsg.queue, skb: last)) { |
389 | skb = skb_queue_next(list: &nn->mbox_cmsg.queue, skb: last); |
390 | space -= 4 + nfp_ccm_mbox_maxlen(skb); |
391 | if (space < 0) |
392 | break; |
393 | last = skb; |
394 | nfp_ccm_mbox_set_busy(skb); |
395 | cnt++; |
396 | if (cnt == NFP_CCM_MBOX_BATCH_LIMIT) |
397 | break; |
398 | } |
399 | spin_unlock_bh(lock: &nn->mbox_cmsg.queue.lock); |
400 | |
401 | /* Now we own all skb's marked in progress, new requests may arrive |
402 | * at the end of the queue. |
403 | */ |
404 | |
405 | nn_ctrl_bar_lock(nn); |
406 | |
407 | nfp_ccm_mbox_copy_in(nn, last); |
408 | |
409 | err = nfp_net_mbox_reconfig(nn, NFP_NET_CFG_MBOX_CMD_TLV_CMSG); |
410 | if (!err) |
411 | nfp_ccm_mbox_copy_out(nn, last); |
412 | else |
413 | nfp_ccm_mbox_mark_all_err(nn, last, err: -EIO); |
414 | |
415 | nn_ctrl_bar_unlock(nn); |
416 | |
417 | wake_up_all(&nn->mbox_cmsg.wq); |
418 | } |
419 | |
420 | static int nfp_ccm_mbox_skb_return(struct sk_buff *skb) |
421 | { |
422 | struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb; |
423 | |
424 | if (cb->err) |
425 | dev_kfree_skb_any(skb); |
426 | return cb->err; |
427 | } |
428 | |
429 | /* If wait timed out but the command is already in progress we have |
430 | * to wait until it finishes. Runners has ownership of the skbs marked |
431 | * as busy. |
432 | */ |
433 | static int |
434 | nfp_ccm_mbox_unlink_unlock(struct nfp_net *nn, struct sk_buff *skb, |
435 | enum nfp_ccm_type type) |
436 | __releases(&nn->mbox_cmsg.queue.lock) |
437 | { |
438 | bool was_first; |
439 | |
440 | if (nfp_ccm_mbox_in_progress(skb)) { |
441 | spin_unlock_bh(lock: &nn->mbox_cmsg.queue.lock); |
442 | |
443 | wait_event(nn->mbox_cmsg.wq, nfp_ccm_mbox_done(skb)); |
444 | smp_rmb(); /* pairs with smp_wmb() after data is written */ |
445 | return nfp_ccm_mbox_skb_return(skb); |
446 | } |
447 | |
448 | was_first = nfp_ccm_mbox_should_run(nn, skb); |
449 | __skb_unlink(skb, list: &nn->mbox_cmsg.queue); |
450 | if (was_first) |
451 | nfp_ccm_mbox_mark_next_runner(nn); |
452 | |
453 | spin_unlock_bh(lock: &nn->mbox_cmsg.queue.lock); |
454 | |
455 | if (was_first) |
456 | wake_up_all(&nn->mbox_cmsg.wq); |
457 | |
458 | nn_dp_warn(&nn->dp, "time out waiting for mbox response to 0x%02x\n" , |
459 | type); |
460 | return -ETIMEDOUT; |
461 | } |
462 | |
463 | static int |
464 | nfp_ccm_mbox_msg_prepare(struct nfp_net *nn, struct sk_buff *skb, |
465 | enum nfp_ccm_type type, |
466 | unsigned int reply_size, unsigned int max_reply_size, |
467 | gfp_t flags) |
468 | { |
469 | const unsigned int mbox_max = nfp_ccm_mbox_max_msg(nn); |
470 | unsigned int max_len; |
471 | ssize_t undersize; |
472 | int err; |
473 | |
474 | if (unlikely(!(nn->tlv_caps.mbox_cmsg_types & BIT(type)))) { |
475 | nn_dp_warn(&nn->dp, |
476 | "message type %d not supported by mailbox\n" , type); |
477 | return -EINVAL; |
478 | } |
479 | |
480 | /* If the reply size is unknown assume it will take the entire |
481 | * mailbox, the callers should do their best for this to never |
482 | * happen. |
483 | */ |
484 | if (!max_reply_size) |
485 | max_reply_size = mbox_max; |
486 | max_reply_size = round_up(max_reply_size, 4); |
487 | |
488 | /* Make sure we can fit the entire reply into the skb, |
489 | * and that we don't have to slow down the mbox handler |
490 | * with allocations. |
491 | */ |
492 | undersize = max_reply_size - (skb_end_pointer(skb) - skb->data); |
493 | if (undersize > 0) { |
494 | err = pskb_expand_head(skb, nhead: 0, ntail: undersize, gfp_mask: flags); |
495 | if (err) { |
496 | nn_dp_warn(&nn->dp, |
497 | "can't allocate reply buffer for mailbox\n" ); |
498 | return err; |
499 | } |
500 | } |
501 | |
502 | /* Make sure that request and response both fit into the mailbox */ |
503 | max_len = max(max_reply_size, round_up(skb->len, 4)); |
504 | if (max_len > mbox_max) { |
505 | nn_dp_warn(&nn->dp, |
506 | "message too big for the mailbox: %u/%u vs %u\n" , |
507 | skb->len, max_reply_size, mbox_max); |
508 | return -EMSGSIZE; |
509 | } |
510 | |
511 | nfp_ccm_mbox_msg_init(skb, exp_reply: reply_size, max_len); |
512 | |
513 | return 0; |
514 | } |
515 | |
516 | static int |
517 | nfp_ccm_mbox_msg_enqueue(struct nfp_net *nn, struct sk_buff *skb, |
518 | enum nfp_ccm_type type, bool critical) |
519 | { |
520 | struct nfp_ccm_hdr *hdr; |
521 | |
522 | assert_spin_locked(&nn->mbox_cmsg.queue.lock); |
523 | |
524 | if (!critical && nn->mbox_cmsg.queue.qlen >= NFP_CCM_MAX_QLEN) { |
525 | nn_dp_warn(&nn->dp, "mailbox request queue too long\n" ); |
526 | return -EBUSY; |
527 | } |
528 | |
529 | hdr = (void *)skb->data; |
530 | hdr->ver = NFP_CCM_ABI_VERSION; |
531 | hdr->type = type; |
532 | hdr->tag = cpu_to_be16(nn->mbox_cmsg.tag++); |
533 | |
534 | __skb_queue_tail(list: &nn->mbox_cmsg.queue, newsk: skb); |
535 | |
536 | return 0; |
537 | } |
538 | |
539 | int __nfp_ccm_mbox_communicate(struct nfp_net *nn, struct sk_buff *skb, |
540 | enum nfp_ccm_type type, |
541 | unsigned int reply_size, |
542 | unsigned int max_reply_size, bool critical) |
543 | { |
544 | int err; |
545 | |
546 | err = nfp_ccm_mbox_msg_prepare(nn, skb, type, reply_size, |
547 | max_reply_size, GFP_KERNEL); |
548 | if (err) |
549 | goto err_free_skb; |
550 | |
551 | spin_lock_bh(lock: &nn->mbox_cmsg.queue.lock); |
552 | |
553 | err = nfp_ccm_mbox_msg_enqueue(nn, skb, type, critical); |
554 | if (err) |
555 | goto err_unlock; |
556 | |
557 | /* First in queue takes the mailbox lock and processes the batch */ |
558 | if (!nfp_ccm_mbox_is_first(nn, skb)) { |
559 | bool to; |
560 | |
561 | spin_unlock_bh(lock: &nn->mbox_cmsg.queue.lock); |
562 | |
563 | to = !wait_event_timeout(nn->mbox_cmsg.wq, |
564 | nfp_ccm_mbox_done(skb) || |
565 | nfp_ccm_mbox_should_run(nn, skb), |
566 | msecs_to_jiffies(NFP_CCM_TIMEOUT)); |
567 | |
568 | /* fast path for those completed by another thread */ |
569 | if (nfp_ccm_mbox_done(skb)) { |
570 | smp_rmb(); /* pairs with wmb after data is written */ |
571 | return nfp_ccm_mbox_skb_return(skb); |
572 | } |
573 | |
574 | spin_lock_bh(lock: &nn->mbox_cmsg.queue.lock); |
575 | |
576 | if (!nfp_ccm_mbox_is_first(nn, skb)) { |
577 | WARN_ON(!to); |
578 | |
579 | err = nfp_ccm_mbox_unlink_unlock(nn, skb, type); |
580 | if (err) |
581 | goto err_free_skb; |
582 | return 0; |
583 | } |
584 | } |
585 | |
586 | /* run queue expects the lock held */ |
587 | nfp_ccm_mbox_run_queue_unlock(nn); |
588 | return nfp_ccm_mbox_skb_return(skb); |
589 | |
590 | err_unlock: |
591 | spin_unlock_bh(lock: &nn->mbox_cmsg.queue.lock); |
592 | err_free_skb: |
593 | dev_kfree_skb_any(skb); |
594 | return err; |
595 | } |
596 | |
597 | int nfp_ccm_mbox_communicate(struct nfp_net *nn, struct sk_buff *skb, |
598 | enum nfp_ccm_type type, |
599 | unsigned int reply_size, |
600 | unsigned int max_reply_size) |
601 | { |
602 | return __nfp_ccm_mbox_communicate(nn, skb, type, reply_size, |
603 | max_reply_size, critical: false); |
604 | } |
605 | |
606 | static void nfp_ccm_mbox_post_runq_work(struct work_struct *work) |
607 | { |
608 | struct sk_buff *skb; |
609 | struct nfp_net *nn; |
610 | |
611 | nn = container_of(work, struct nfp_net, mbox_cmsg.runq_work); |
612 | |
613 | spin_lock_bh(lock: &nn->mbox_cmsg.queue.lock); |
614 | |
615 | skb = __skb_peek(list_: &nn->mbox_cmsg.queue); |
616 | if (WARN_ON(!skb || !nfp_ccm_mbox_is_posted(skb) || |
617 | !nfp_ccm_mbox_should_run(nn, skb))) { |
618 | spin_unlock_bh(lock: &nn->mbox_cmsg.queue.lock); |
619 | return; |
620 | } |
621 | |
622 | nfp_ccm_mbox_run_queue_unlock(nn); |
623 | } |
624 | |
625 | static void nfp_ccm_mbox_post_wait_work(struct work_struct *work) |
626 | { |
627 | struct sk_buff *skb; |
628 | struct nfp_net *nn; |
629 | int err; |
630 | |
631 | nn = container_of(work, struct nfp_net, mbox_cmsg.wait_work); |
632 | |
633 | skb = skb_peek(list_: &nn->mbox_cmsg.queue); |
634 | if (WARN_ON(!skb || !nfp_ccm_mbox_is_posted(skb))) |
635 | /* Should never happen so it's unclear what to do here.. */ |
636 | goto exit_unlock_wake; |
637 | |
638 | err = nfp_net_mbox_reconfig_wait_posted(nn); |
639 | if (!err) |
640 | nfp_ccm_mbox_copy_out(nn, last: skb); |
641 | else |
642 | nfp_ccm_mbox_mark_all_err(nn, last: skb, err: -EIO); |
643 | exit_unlock_wake: |
644 | nn_ctrl_bar_unlock(nn); |
645 | wake_up_all(&nn->mbox_cmsg.wq); |
646 | } |
647 | |
648 | int nfp_ccm_mbox_post(struct nfp_net *nn, struct sk_buff *skb, |
649 | enum nfp_ccm_type type, unsigned int max_reply_size) |
650 | { |
651 | int err; |
652 | |
653 | err = nfp_ccm_mbox_msg_prepare(nn, skb, type, reply_size: 0, max_reply_size, |
654 | GFP_ATOMIC); |
655 | if (err) |
656 | goto err_free_skb; |
657 | |
658 | nfp_ccm_mbox_mark_posted(skb); |
659 | |
660 | spin_lock_bh(lock: &nn->mbox_cmsg.queue.lock); |
661 | |
662 | err = nfp_ccm_mbox_msg_enqueue(nn, skb, type, critical: false); |
663 | if (err) |
664 | goto err_unlock; |
665 | |
666 | if (nfp_ccm_mbox_is_first(nn, skb)) { |
667 | if (nn_ctrl_bar_trylock(nn)) { |
668 | nfp_ccm_mbox_copy_in(nn, last: skb); |
669 | nfp_net_mbox_reconfig_post(nn, |
670 | NFP_NET_CFG_MBOX_CMD_TLV_CMSG); |
671 | queue_work(wq: nn->mbox_cmsg.workq, |
672 | work: &nn->mbox_cmsg.wait_work); |
673 | } else { |
674 | nfp_ccm_mbox_mark_next_runner(nn); |
675 | } |
676 | } |
677 | |
678 | spin_unlock_bh(lock: &nn->mbox_cmsg.queue.lock); |
679 | |
680 | return 0; |
681 | |
682 | err_unlock: |
683 | spin_unlock_bh(lock: &nn->mbox_cmsg.queue.lock); |
684 | err_free_skb: |
685 | dev_kfree_skb_any(skb); |
686 | return err; |
687 | } |
688 | |
689 | struct sk_buff * |
690 | nfp_ccm_mbox_msg_alloc(struct nfp_net *nn, unsigned int req_size, |
691 | unsigned int reply_size, gfp_t flags) |
692 | { |
693 | unsigned int max_size; |
694 | struct sk_buff *skb; |
695 | |
696 | if (!reply_size) |
697 | max_size = nfp_ccm_mbox_max_msg(nn); |
698 | else |
699 | max_size = max(req_size, reply_size); |
700 | max_size = round_up(max_size, 4); |
701 | |
702 | skb = alloc_skb(size: max_size, priority: flags); |
703 | if (!skb) |
704 | return NULL; |
705 | |
706 | skb_put(skb, len: req_size); |
707 | |
708 | return skb; |
709 | } |
710 | |
711 | bool nfp_ccm_mbox_fits(struct nfp_net *nn, unsigned int size) |
712 | { |
713 | return nfp_ccm_mbox_max_msg(nn) >= size; |
714 | } |
715 | |
716 | int nfp_ccm_mbox_init(struct nfp_net *nn) |
717 | { |
718 | return 0; |
719 | } |
720 | |
721 | void nfp_ccm_mbox_clean(struct nfp_net *nn) |
722 | { |
723 | drain_workqueue(wq: nn->mbox_cmsg.workq); |
724 | } |
725 | |
726 | int nfp_ccm_mbox_alloc(struct nfp_net *nn) |
727 | { |
728 | skb_queue_head_init(list: &nn->mbox_cmsg.queue); |
729 | init_waitqueue_head(&nn->mbox_cmsg.wq); |
730 | INIT_WORK(&nn->mbox_cmsg.wait_work, nfp_ccm_mbox_post_wait_work); |
731 | INIT_WORK(&nn->mbox_cmsg.runq_work, nfp_ccm_mbox_post_runq_work); |
732 | |
733 | nn->mbox_cmsg.workq = alloc_workqueue(fmt: "nfp-ccm-mbox" , flags: WQ_UNBOUND, max_active: 0); |
734 | if (!nn->mbox_cmsg.workq) |
735 | return -ENOMEM; |
736 | return 0; |
737 | } |
738 | |
739 | void nfp_ccm_mbox_free(struct nfp_net *nn) |
740 | { |
741 | destroy_workqueue(wq: nn->mbox_cmsg.workq); |
742 | WARN_ON(!skb_queue_empty(&nn->mbox_cmsg.queue)); |
743 | } |
744 | |