1 | // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause |
2 | /* |
3 | * Copyright(c) 2016 - 2018 Intel Corporation. |
4 | */ |
5 | |
6 | #include "hfi.h" |
7 | #include "verbs_txreq.h" |
8 | #include "qp.h" |
9 | #include "trace.h" |
10 | |
11 | #define TXREQ_LEN 24 |
12 | |
13 | void hfi1_put_txreq(struct verbs_txreq *tx) |
14 | { |
15 | struct hfi1_ibdev *dev; |
16 | struct rvt_qp *qp; |
17 | unsigned long flags; |
18 | unsigned int seq; |
19 | struct hfi1_qp_priv *priv; |
20 | |
21 | qp = tx->qp; |
22 | dev = to_idev(ibdev: qp->ibqp.device); |
23 | |
24 | if (tx->mr) |
25 | rvt_put_mr(mr: tx->mr); |
26 | |
27 | sdma_txclean(dd: dd_from_dev(dev), tx: &tx->txreq); |
28 | |
29 | /* Free verbs_txreq and return to slab cache */ |
30 | kmem_cache_free(s: dev->verbs_txreq_cache, objp: tx); |
31 | |
32 | do { |
33 | seq = read_seqbegin(sl: &dev->txwait_lock); |
34 | if (!list_empty(head: &dev->txwait)) { |
35 | struct iowait *wait; |
36 | |
37 | write_seqlock_irqsave(&dev->txwait_lock, flags); |
38 | wait = list_first_entry(&dev->txwait, struct iowait, |
39 | list); |
40 | qp = iowait_to_qp(s_iowait: wait); |
41 | priv = qp->priv; |
42 | list_del_init(entry: &priv->s_iowait.list); |
43 | /* refcount held until actual wake up */ |
44 | write_sequnlock_irqrestore(sl: &dev->txwait_lock, flags); |
45 | hfi1_qp_wakeup(qp, RVT_S_WAIT_TX); |
46 | break; |
47 | } |
48 | } while (read_seqretry(sl: &dev->txwait_lock, start: seq)); |
49 | } |
50 | |
51 | struct verbs_txreq *__get_txreq(struct hfi1_ibdev *dev, |
52 | struct rvt_qp *qp) |
53 | __must_hold(&qp->s_lock) |
54 | { |
55 | struct verbs_txreq *tx = NULL; |
56 | |
57 | write_seqlock(sl: &dev->txwait_lock); |
58 | if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) { |
59 | struct hfi1_qp_priv *priv; |
60 | |
61 | tx = kmem_cache_alloc(cachep: dev->verbs_txreq_cache, VERBS_TXREQ_GFP); |
62 | if (tx) |
63 | goto out; |
64 | priv = qp->priv; |
65 | if (list_empty(head: &priv->s_iowait.list)) { |
66 | dev->n_txwait++; |
67 | qp->s_flags |= RVT_S_WAIT_TX; |
68 | list_add_tail(new: &priv->s_iowait.list, head: &dev->txwait); |
69 | priv->s_iowait.lock = &dev->txwait_lock; |
70 | trace_hfi1_qpsleep(qp, RVT_S_WAIT_TX); |
71 | rvt_get_qp(qp); |
72 | } |
73 | qp->s_flags &= ~RVT_S_BUSY; |
74 | } |
75 | out: |
76 | write_sequnlock(sl: &dev->txwait_lock); |
77 | return tx; |
78 | } |
79 | |
80 | int verbs_txreq_init(struct hfi1_ibdev *dev) |
81 | { |
82 | char buf[TXREQ_LEN]; |
83 | struct hfi1_devdata *dd = dd_from_dev(dev); |
84 | |
85 | snprintf(buf, size: sizeof(buf), fmt: "hfi1_%u_vtxreq_cache" , dd->unit); |
86 | dev->verbs_txreq_cache = kmem_cache_create(name: buf, |
87 | size: sizeof(struct verbs_txreq), |
88 | align: 0, SLAB_HWCACHE_ALIGN, |
89 | NULL); |
90 | if (!dev->verbs_txreq_cache) |
91 | return -ENOMEM; |
92 | return 0; |
93 | } |
94 | |
95 | void verbs_txreq_exit(struct hfi1_ibdev *dev) |
96 | { |
97 | kmem_cache_destroy(s: dev->verbs_txreq_cache); |
98 | dev->verbs_txreq_cache = NULL; |
99 | } |
100 | |