1 | /* |
2 | * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved. |
3 | * |
4 | * This software is available to you under a choice of one of two |
5 | * licenses. You may choose to be licensed under the terms of the GNU |
6 | * General Public License (GPL) Version 2, available from the file |
7 | * COPYING in the main directory of this source tree, or the |
8 | * OpenIB.org BSD license below: |
9 | * |
10 | * Redistribution and use in source and binary forms, with or |
11 | * without modification, are permitted provided that the following |
12 | * conditions are met: |
13 | * |
14 | * - Redistributions of source code must retain the above |
15 | * copyright notice, this list of conditions and the following |
16 | * disclaimer. |
17 | * |
18 | * - Redistributions in binary form must reproduce the above |
19 | * copyright notice, this list of conditions and the following |
20 | * disclaimer in the documentation and/or other materials |
21 | * provided with the distribution. |
22 | * |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
30 | * SOFTWARE. |
31 | */ |
32 | |
33 | #ifndef __MLX5_WQ_H__ |
34 | #define __MLX5_WQ_H__ |
35 | |
36 | #include <linux/mlx5/mlx5_ifc.h> |
37 | #include <linux/mlx5/cq.h> |
38 | #include <linux/mlx5/qp.h> |
39 | |
40 | struct mlx5_wq_param { |
41 | int buf_numa_node; |
42 | int db_numa_node; |
43 | }; |
44 | |
45 | struct mlx5_wq_ctrl { |
46 | struct mlx5_core_dev *mdev; |
47 | struct mlx5_frag_buf buf; |
48 | struct mlx5_db db; |
49 | }; |
50 | |
51 | struct mlx5_wq_cyc { |
52 | struct mlx5_frag_buf_ctrl fbc; |
53 | __be32 *db; |
54 | u16 sz; |
55 | u16 wqe_ctr; |
56 | u16 cur_sz; |
57 | }; |
58 | |
59 | struct mlx5_wq_qp { |
60 | struct mlx5_wq_cyc rq; |
61 | struct mlx5_wq_cyc sq; |
62 | }; |
63 | |
64 | struct mlx5_cqwq { |
65 | struct mlx5_frag_buf_ctrl fbc; |
66 | __be32 *db; |
67 | u32 cc; /* consumer counter */ |
68 | }; |
69 | |
70 | struct mlx5_wq_ll { |
71 | struct mlx5_frag_buf_ctrl fbc; |
72 | __be32 *db; |
73 | __be16 *tail_next; |
74 | u16 head; |
75 | u16 wqe_ctr; |
76 | u16 cur_sz; |
77 | }; |
78 | |
79 | int mlx5_wq_cyc_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param, |
80 | void *wqc, struct mlx5_wq_cyc *wq, |
81 | struct mlx5_wq_ctrl *wq_ctrl); |
82 | void mlx5_wq_cyc_wqe_dump(struct mlx5_wq_cyc *wq, u16 ix, u8 nstrides); |
83 | void mlx5_wq_cyc_reset(struct mlx5_wq_cyc *wq); |
84 | |
85 | int mlx5_wq_qp_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param, |
86 | void *qpc, struct mlx5_wq_qp *wq, |
87 | struct mlx5_wq_ctrl *wq_ctrl); |
88 | |
89 | int mlx5_cqwq_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param, |
90 | void *cqc, struct mlx5_cqwq *wq, |
91 | struct mlx5_wq_ctrl *wq_ctrl); |
92 | |
93 | int mlx5_wq_ll_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param, |
94 | void *wqc, struct mlx5_wq_ll *wq, |
95 | struct mlx5_wq_ctrl *wq_ctrl); |
96 | void mlx5_wq_ll_reset(struct mlx5_wq_ll *wq); |
97 | |
98 | void mlx5_wq_destroy(struct mlx5_wq_ctrl *wq_ctrl); |
99 | |
100 | static inline u32 mlx5_wq_cyc_get_size(struct mlx5_wq_cyc *wq) |
101 | { |
102 | return (u32)wq->fbc.sz_m1 + 1; |
103 | } |
104 | |
105 | static inline int mlx5_wq_cyc_is_full(struct mlx5_wq_cyc *wq) |
106 | { |
107 | return wq->cur_sz == wq->sz; |
108 | } |
109 | |
110 | static inline int mlx5_wq_cyc_missing(struct mlx5_wq_cyc *wq) |
111 | { |
112 | return wq->sz - wq->cur_sz; |
113 | } |
114 | |
115 | static inline int mlx5_wq_cyc_is_empty(struct mlx5_wq_cyc *wq) |
116 | { |
117 | return !wq->cur_sz; |
118 | } |
119 | |
120 | static inline void mlx5_wq_cyc_push(struct mlx5_wq_cyc *wq) |
121 | { |
122 | wq->wqe_ctr++; |
123 | wq->cur_sz++; |
124 | } |
125 | |
126 | static inline void mlx5_wq_cyc_push_n(struct mlx5_wq_cyc *wq, u16 n) |
127 | { |
128 | wq->wqe_ctr += n; |
129 | wq->cur_sz += n; |
130 | } |
131 | |
132 | static inline void mlx5_wq_cyc_pop(struct mlx5_wq_cyc *wq) |
133 | { |
134 | wq->cur_sz--; |
135 | } |
136 | |
137 | static inline void mlx5_wq_cyc_update_db_record(struct mlx5_wq_cyc *wq) |
138 | { |
139 | *wq->db = cpu_to_be32(wq->wqe_ctr); |
140 | } |
141 | |
142 | static inline u16 mlx5_wq_cyc_ctr2ix(struct mlx5_wq_cyc *wq, u16 ctr) |
143 | { |
144 | return ctr & wq->fbc.sz_m1; |
145 | } |
146 | |
147 | static inline u16 mlx5_wq_cyc_get_head(struct mlx5_wq_cyc *wq) |
148 | { |
149 | return mlx5_wq_cyc_ctr2ix(wq, ctr: wq->wqe_ctr); |
150 | } |
151 | |
152 | static inline u16 mlx5_wq_cyc_get_tail(struct mlx5_wq_cyc *wq) |
153 | { |
154 | return mlx5_wq_cyc_ctr2ix(wq, ctr: wq->wqe_ctr - wq->cur_sz); |
155 | } |
156 | |
157 | static inline void *mlx5_wq_cyc_get_wqe(struct mlx5_wq_cyc *wq, u16 ix) |
158 | { |
159 | return mlx5_frag_buf_get_wqe(fbc: &wq->fbc, ix); |
160 | } |
161 | |
162 | static inline u16 mlx5_wq_cyc_get_contig_wqebbs(struct mlx5_wq_cyc *wq, u16 ix) |
163 | { |
164 | return mlx5_frag_buf_get_idx_last_contig_stride(fbc: &wq->fbc, ix) - ix + 1; |
165 | } |
166 | |
167 | static inline int mlx5_wq_cyc_cc_bigger(u16 cc1, u16 cc2) |
168 | { |
169 | int equal = (cc1 == cc2); |
170 | int smaller = 0x8000 & (cc1 - cc2); |
171 | |
172 | return !equal && !smaller; |
173 | } |
174 | |
175 | static inline u16 mlx5_wq_cyc_get_counter(struct mlx5_wq_cyc *wq) |
176 | { |
177 | return wq->wqe_ctr; |
178 | } |
179 | |
180 | static inline u32 mlx5_cqwq_get_size(struct mlx5_cqwq *wq) |
181 | { |
182 | return wq->fbc.sz_m1 + 1; |
183 | } |
184 | |
185 | static inline u8 mlx5_cqwq_get_log_stride_size(struct mlx5_cqwq *wq) |
186 | { |
187 | return wq->fbc.log_stride; |
188 | } |
189 | |
190 | static inline u32 mlx5_cqwq_ctr2ix(struct mlx5_cqwq *wq, u32 ctr) |
191 | { |
192 | return ctr & wq->fbc.sz_m1; |
193 | } |
194 | |
195 | static inline u32 mlx5_cqwq_get_ci(struct mlx5_cqwq *wq) |
196 | { |
197 | return mlx5_cqwq_ctr2ix(wq, ctr: wq->cc); |
198 | } |
199 | |
200 | static inline struct mlx5_cqe64 *mlx5_cqwq_get_wqe(struct mlx5_cqwq *wq, u32 ix) |
201 | { |
202 | struct mlx5_cqe64 *cqe = mlx5_frag_buf_get_wqe(fbc: &wq->fbc, ix); |
203 | |
204 | /* For 128B CQEs the data is in the last 64B */ |
205 | cqe += wq->fbc.log_stride == 7; |
206 | |
207 | return cqe; |
208 | } |
209 | |
210 | static inline u32 mlx5_cqwq_get_ctr_wrap_cnt(struct mlx5_cqwq *wq, u32 ctr) |
211 | { |
212 | return ctr >> wq->fbc.log_sz; |
213 | } |
214 | |
215 | static inline u32 mlx5_cqwq_get_wrap_cnt(struct mlx5_cqwq *wq) |
216 | { |
217 | return mlx5_cqwq_get_ctr_wrap_cnt(wq, ctr: wq->cc); |
218 | } |
219 | |
220 | static inline void mlx5_cqwq_pop(struct mlx5_cqwq *wq) |
221 | { |
222 | wq->cc++; |
223 | } |
224 | |
225 | static inline void mlx5_cqwq_update_db_record(struct mlx5_cqwq *wq) |
226 | { |
227 | *wq->db = cpu_to_be32(wq->cc & 0xffffff); |
228 | } |
229 | |
230 | static inline struct mlx5_cqe64 *mlx5_cqwq_get_cqe(struct mlx5_cqwq *wq) |
231 | { |
232 | u32 ci = mlx5_cqwq_get_ci(wq); |
233 | struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(wq, ix: ci); |
234 | u8 cqe_ownership_bit = cqe->op_own & MLX5_CQE_OWNER_MASK; |
235 | u8 sw_ownership_val = mlx5_cqwq_get_wrap_cnt(wq) & 1; |
236 | |
237 | if (cqe_ownership_bit != sw_ownership_val) |
238 | return NULL; |
239 | |
240 | /* ensure cqe content is read after cqe ownership bit */ |
241 | dma_rmb(); |
242 | |
243 | return cqe; |
244 | } |
245 | |
246 | static inline |
247 | struct mlx5_cqe64 *mlx5_cqwq_get_cqe_enahnced_comp(struct mlx5_cqwq *wq) |
248 | { |
249 | u8 sw_validity_iteration_count = mlx5_cqwq_get_wrap_cnt(wq) & 0xff; |
250 | u32 ci = mlx5_cqwq_get_ci(wq); |
251 | struct mlx5_cqe64 *cqe; |
252 | |
253 | cqe = mlx5_cqwq_get_wqe(wq, ix: ci); |
254 | if (cqe->validity_iteration_count != sw_validity_iteration_count) |
255 | return NULL; |
256 | |
257 | /* ensure cqe content is read after cqe ownership bit/validity byte */ |
258 | dma_rmb(); |
259 | |
260 | return cqe; |
261 | } |
262 | |
263 | static inline u32 mlx5_wq_ll_get_size(struct mlx5_wq_ll *wq) |
264 | { |
265 | return (u32)wq->fbc.sz_m1 + 1; |
266 | } |
267 | |
268 | static inline int mlx5_wq_ll_is_full(struct mlx5_wq_ll *wq) |
269 | { |
270 | return wq->cur_sz == wq->fbc.sz_m1; |
271 | } |
272 | |
273 | static inline int mlx5_wq_ll_is_empty(struct mlx5_wq_ll *wq) |
274 | { |
275 | return !wq->cur_sz; |
276 | } |
277 | |
278 | static inline int mlx5_wq_ll_missing(struct mlx5_wq_ll *wq) |
279 | { |
280 | return wq->fbc.sz_m1 - wq->cur_sz; |
281 | } |
282 | |
283 | static inline void *mlx5_wq_ll_get_wqe(struct mlx5_wq_ll *wq, u16 ix) |
284 | { |
285 | return mlx5_frag_buf_get_wqe(fbc: &wq->fbc, ix); |
286 | } |
287 | |
288 | static inline u16 mlx5_wq_ll_get_wqe_next_ix(struct mlx5_wq_ll *wq, u16 ix) |
289 | { |
290 | struct mlx5_wqe_srq_next_seg *wqe = mlx5_wq_ll_get_wqe(wq, ix); |
291 | |
292 | return be16_to_cpu(wqe->next_wqe_index); |
293 | } |
294 | |
295 | static inline void mlx5_wq_ll_push(struct mlx5_wq_ll *wq, u16 head_next) |
296 | { |
297 | wq->head = head_next; |
298 | wq->wqe_ctr++; |
299 | wq->cur_sz++; |
300 | } |
301 | |
302 | static inline void mlx5_wq_ll_pop(struct mlx5_wq_ll *wq, __be16 ix, |
303 | __be16 *next_tail_next) |
304 | { |
305 | *wq->tail_next = ix; |
306 | wq->tail_next = next_tail_next; |
307 | wq->cur_sz--; |
308 | } |
309 | |
310 | static inline void mlx5_wq_ll_update_db_record(struct mlx5_wq_ll *wq) |
311 | { |
312 | *wq->db = cpu_to_be32(wq->wqe_ctr); |
313 | } |
314 | |
315 | static inline u16 mlx5_wq_ll_get_head(struct mlx5_wq_ll *wq) |
316 | { |
317 | return wq->head; |
318 | } |
319 | |
320 | static inline u16 mlx5_wq_ll_get_counter(struct mlx5_wq_ll *wq) |
321 | { |
322 | return wq->wqe_ctr; |
323 | } |
324 | |
325 | #endif /* __MLX5_WQ_H__ */ |
326 | |