1 | // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause |
2 | /* |
3 | * Copyright(c) 2016 Intel Corporation. |
4 | */ |
5 | |
6 | #include <rdma/rdmavt_qp.h> |
7 | #include <rdma/ib_hdrs.h> |
8 | |
9 | /* |
10 | * Convert the AETH credit code into the number of credits. |
11 | */ |
12 | static const u16 credit_table[31] = { |
13 | 0, /* 0 */ |
14 | 1, /* 1 */ |
15 | 2, /* 2 */ |
16 | 3, /* 3 */ |
17 | 4, /* 4 */ |
18 | 6, /* 5 */ |
19 | 8, /* 6 */ |
20 | 12, /* 7 */ |
21 | 16, /* 8 */ |
22 | 24, /* 9 */ |
23 | 32, /* A */ |
24 | 48, /* B */ |
25 | 64, /* C */ |
26 | 96, /* D */ |
27 | 128, /* E */ |
28 | 192, /* F */ |
29 | 256, /* 10 */ |
30 | 384, /* 11 */ |
31 | 512, /* 12 */ |
32 | 768, /* 13 */ |
33 | 1024, /* 14 */ |
34 | 1536, /* 15 */ |
35 | 2048, /* 16 */ |
36 | 3072, /* 17 */ |
37 | 4096, /* 18 */ |
38 | 6144, /* 19 */ |
39 | 8192, /* 1A */ |
40 | 12288, /* 1B */ |
41 | 16384, /* 1C */ |
42 | 24576, /* 1D */ |
43 | 32768 /* 1E */ |
44 | }; |
45 | |
46 | /** |
47 | * rvt_compute_aeth - compute the AETH (syndrome + MSN) |
48 | * @qp: the queue pair to compute the AETH for |
49 | * |
50 | * Returns the AETH. |
51 | */ |
52 | __be32 rvt_compute_aeth(struct rvt_qp *qp) |
53 | { |
54 | u32 aeth = qp->r_msn & IB_MSN_MASK; |
55 | |
56 | if (qp->ibqp.srq) { |
57 | /* |
58 | * Shared receive queues don't generate credits. |
59 | * Set the credit field to the invalid value. |
60 | */ |
61 | aeth |= IB_AETH_CREDIT_INVAL << IB_AETH_CREDIT_SHIFT; |
62 | } else { |
63 | u32 min, max, x; |
64 | u32 credits; |
65 | u32 head; |
66 | u32 tail; |
67 | |
68 | credits = READ_ONCE(qp->r_rq.kwq->count); |
69 | if (credits == 0) { |
70 | /* sanity check pointers before trusting them */ |
71 | if (qp->ip) { |
72 | head = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->head); |
73 | tail = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->tail); |
74 | } else { |
75 | head = READ_ONCE(qp->r_rq.kwq->head); |
76 | tail = READ_ONCE(qp->r_rq.kwq->tail); |
77 | } |
78 | if (head >= qp->r_rq.size) |
79 | head = 0; |
80 | if (tail >= qp->r_rq.size) |
81 | tail = 0; |
82 | /* |
83 | * Compute the number of credits available (RWQEs). |
84 | * There is a small chance that the pair of reads are |
85 | * not atomic, which is OK, since the fuzziness is |
86 | * resolved as further ACKs go out. |
87 | */ |
88 | credits = rvt_get_rq_count(rq: &qp->r_rq, head, tail); |
89 | } |
90 | /* |
91 | * Binary search the credit table to find the code to |
92 | * use. |
93 | */ |
94 | min = 0; |
95 | max = 31; |
96 | for (;;) { |
97 | x = (min + max) / 2; |
98 | if (credit_table[x] == credits) |
99 | break; |
100 | if (credit_table[x] > credits) { |
101 | max = x; |
102 | } else { |
103 | if (min == x) |
104 | break; |
105 | min = x; |
106 | } |
107 | } |
108 | aeth |= x << IB_AETH_CREDIT_SHIFT; |
109 | } |
110 | return cpu_to_be32(aeth); |
111 | } |
112 | EXPORT_SYMBOL(rvt_compute_aeth); |
113 | |
114 | /** |
115 | * rvt_get_credit - flush the send work queue of a QP |
116 | * @qp: the qp who's send work queue to flush |
117 | * @aeth: the Acknowledge Extended Transport Header |
118 | * |
119 | * The QP s_lock should be held. |
120 | */ |
121 | void rvt_get_credit(struct rvt_qp *qp, u32 aeth) |
122 | { |
123 | struct rvt_dev_info *rdi = ib_to_rvt(ibdev: qp->ibqp.device); |
124 | u32 credit = (aeth >> IB_AETH_CREDIT_SHIFT) & IB_AETH_CREDIT_MASK; |
125 | |
126 | lockdep_assert_held(&qp->s_lock); |
127 | /* |
128 | * If the credit is invalid, we can send |
129 | * as many packets as we like. Otherwise, we have to |
130 | * honor the credit field. |
131 | */ |
132 | if (credit == IB_AETH_CREDIT_INVAL) { |
133 | if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) { |
134 | qp->s_flags |= RVT_S_UNLIMITED_CREDIT; |
135 | if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) { |
136 | qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT; |
137 | rdi->driver_f.schedule_send(qp); |
138 | } |
139 | } |
140 | } else if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) { |
141 | /* Compute new LSN (i.e., MSN + credit) */ |
142 | credit = (aeth + credit_table[credit]) & IB_MSN_MASK; |
143 | if (rvt_cmp_msn(a: credit, b: qp->s_lsn) > 0) { |
144 | qp->s_lsn = credit; |
145 | if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) { |
146 | qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT; |
147 | rdi->driver_f.schedule_send(qp); |
148 | } |
149 | } |
150 | } |
151 | } |
152 | EXPORT_SYMBOL(rvt_get_credit); |
153 | |
154 | /** |
155 | * rvt_restart_sge - rewind the sge state for a wqe |
156 | * @ss: the sge state pointer |
157 | * @wqe: the wqe to rewind |
158 | * @len: the data length from the start of the wqe in bytes |
159 | * |
160 | * Returns the remaining data length. |
161 | */ |
162 | u32 rvt_restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe, u32 len) |
163 | { |
164 | ss->sge = wqe->sg_list[0]; |
165 | ss->sg_list = wqe->sg_list + 1; |
166 | ss->num_sge = wqe->wr.num_sge; |
167 | ss->total_len = wqe->length; |
168 | rvt_skip_sge(ss, length: len, release: false); |
169 | return wqe->length - len; |
170 | } |
171 | EXPORT_SYMBOL(rvt_restart_sge); |
172 | |
173 | |