1 | /* |
2 | * Broadcom NetXtreme-E RoCE driver. |
3 | * |
4 | * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term |
5 | * Broadcom refers to Broadcom Limited and/or its subsidiaries. |
6 | * |
7 | * This software is available to you under a choice of one of two |
8 | * licenses. You may choose to be licensed under the terms of the GNU |
9 | * General Public License (GPL) Version 2, available from the file |
10 | * COPYING in the main directory of this source tree, or the |
11 | * BSD license below: |
12 | * |
13 | * Redistribution and use in source and binary forms, with or without |
14 | * modification, are permitted provided that the following conditions |
15 | * are met: |
16 | * |
17 | * 1. Redistributions of source code must retain the above copyright |
18 | * notice, this list of conditions and the following disclaimer. |
19 | * 2. Redistributions in binary form must reproduce the above copyright |
20 | * notice, this list of conditions and the following disclaimer in |
21 | * the documentation and/or other materials provided with the |
22 | * distribution. |
23 | * |
24 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' |
25 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, |
26 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
27 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS |
28 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
29 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
30 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR |
31 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, |
32 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE |
33 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN |
34 | * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
35 | * |
36 | * Description: Fast Path Operators (header) |
37 | */ |
38 | |
39 | #ifndef __BNXT_QPLIB_FP_H__ |
40 | #define __BNXT_QPLIB_FP_H__ |
41 | |
42 | #include <rdma/bnxt_re-abi.h> |
43 | |
44 | /* Few helper structures temporarily defined here |
45 | * should get rid of these when roce_hsi.h is updated |
46 | * in original code base |
47 | */ |
48 | struct sq_ud_ext_hdr { |
49 | __le32 dst_qp; |
50 | __le32 avid; |
51 | __le64 rsvd; |
52 | }; |
53 | |
54 | struct sq_raw_ext_hdr { |
55 | __le32 cfa_meta; |
56 | __le32 rsvd0; |
57 | __le64 rsvd1; |
58 | }; |
59 | |
60 | struct sq_rdma_ext_hdr { |
61 | __le64 remote_va; |
62 | __le32 remote_key; |
63 | __le32 rsvd; |
64 | }; |
65 | |
66 | struct sq_atomic_ext_hdr { |
67 | __le64 swap_data; |
68 | __le64 cmp_data; |
69 | }; |
70 | |
71 | struct sq_fr_pmr_ext_hdr { |
72 | __le64 pblptr; |
73 | __le64 va; |
74 | }; |
75 | |
76 | struct sq_bind_ext_hdr { |
77 | __le64 va; |
78 | __le32 length_lo; |
79 | __le32 length_hi; |
80 | }; |
81 | |
82 | struct rq_ext_hdr { |
83 | __le64 rsvd1; |
84 | __le64 rsvd2; |
85 | }; |
86 | |
87 | /* Helper structures end */ |
88 | |
89 | struct bnxt_qplib_srq { |
90 | struct bnxt_qplib_pd *pd; |
91 | struct bnxt_qplib_dpi *dpi; |
92 | struct bnxt_qplib_db_info dbinfo; |
93 | u64 srq_handle; |
94 | u32 id; |
95 | u16 wqe_size; |
96 | u32 max_wqe; |
97 | u32 max_sge; |
98 | u32 threshold; |
99 | bool arm_req; |
100 | struct bnxt_qplib_cq *cq; |
101 | struct bnxt_qplib_hwq hwq; |
102 | struct bnxt_qplib_swq *swq; |
103 | int start_idx; |
104 | int last_idx; |
105 | struct bnxt_qplib_sg_info sg_info; |
106 | u16 eventq_hw_ring_id; |
107 | spinlock_t lock; /* protect SRQE link list */ |
108 | }; |
109 | |
110 | struct bnxt_qplib_sge { |
111 | u64 addr; |
112 | u32 lkey; |
113 | u32 size; |
114 | }; |
115 | |
116 | #define BNXT_QPLIB_QP_MAX_SGL 6 |
117 | struct bnxt_qplib_swq { |
118 | u64 wr_id; |
119 | int next_idx; |
120 | u8 type; |
121 | u8 flags; |
122 | u32 start_psn; |
123 | u32 next_psn; |
124 | u32 slot_idx; |
125 | u8 slots; |
126 | struct sq_psn_search *psn_search; |
127 | struct sq_psn_search_ext *psn_ext; |
128 | }; |
129 | |
130 | struct bnxt_qplib_swqe { |
131 | /* General */ |
132 | #define BNXT_QPLIB_FENCE_WRID 0x46454E43 /* "FENC" */ |
133 | u64 wr_id; |
134 | u8 reqs_type; |
135 | u8 type; |
136 | #define BNXT_QPLIB_SWQE_TYPE_SEND 0 |
137 | #define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM 1 |
138 | #define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV 2 |
139 | #define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE 4 |
140 | #define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM 5 |
141 | #define BNXT_QPLIB_SWQE_TYPE_RDMA_READ 6 |
142 | #define BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP 8 |
143 | #define BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD 11 |
144 | #define BNXT_QPLIB_SWQE_TYPE_LOCAL_INV 12 |
145 | #define BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR 13 |
146 | #define BNXT_QPLIB_SWQE_TYPE_REG_MR 13 |
147 | #define BNXT_QPLIB_SWQE_TYPE_BIND_MW 14 |
148 | #define BNXT_QPLIB_SWQE_TYPE_RECV 128 |
149 | #define BNXT_QPLIB_SWQE_TYPE_RECV_RDMA_IMM 129 |
150 | u8 flags; |
151 | #define BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP BIT(0) |
152 | #define BNXT_QPLIB_SWQE_FLAGS_RD_ATOMIC_FENCE BIT(1) |
153 | #define BNXT_QPLIB_SWQE_FLAGS_UC_FENCE BIT(2) |
154 | #define BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT BIT(3) |
155 | #define BNXT_QPLIB_SWQE_FLAGS_INLINE BIT(4) |
156 | struct bnxt_qplib_sge sg_list[BNXT_QPLIB_QP_MAX_SGL]; |
157 | int num_sge; |
158 | /* Max inline data is 96 bytes */ |
159 | u32 inline_len; |
160 | #define BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH 96 |
161 | u8 inline_data[BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH]; |
162 | |
163 | union { |
164 | /* Send, with imm, inval key */ |
165 | struct { |
166 | union { |
167 | __be32 imm_data; |
168 | u32 inv_key; |
169 | }; |
170 | u32 q_key; |
171 | u32 dst_qp; |
172 | u16 avid; |
173 | } send; |
174 | |
175 | /* Send Raw Ethernet and QP1 */ |
176 | struct { |
177 | u16 lflags; |
178 | u16 cfa_action; |
179 | u32 cfa_meta; |
180 | } rawqp1; |
181 | |
182 | /* RDMA write, with imm, read */ |
183 | struct { |
184 | union { |
185 | __be32 imm_data; |
186 | u32 inv_key; |
187 | }; |
188 | u64 remote_va; |
189 | u32 r_key; |
190 | } rdma; |
191 | |
192 | /* Atomic cmp/swap, fetch/add */ |
193 | struct { |
194 | u64 remote_va; |
195 | u32 r_key; |
196 | u64 swap_data; |
197 | u64 cmp_data; |
198 | } atomic; |
199 | |
200 | /* Local Invalidate */ |
201 | struct { |
202 | u32 inv_l_key; |
203 | } local_inv; |
204 | |
205 | /* FR-PMR */ |
206 | struct { |
207 | u8 access_cntl; |
208 | u8 pg_sz_log; |
209 | bool zero_based; |
210 | u32 l_key; |
211 | u32 length; |
212 | u8 pbl_pg_sz_log; |
213 | #define BNXT_QPLIB_SWQE_PAGE_SIZE_4K 0 |
214 | #define BNXT_QPLIB_SWQE_PAGE_SIZE_8K 1 |
215 | #define BNXT_QPLIB_SWQE_PAGE_SIZE_64K 4 |
216 | #define BNXT_QPLIB_SWQE_PAGE_SIZE_256K 6 |
217 | #define BNXT_QPLIB_SWQE_PAGE_SIZE_1M 8 |
218 | #define BNXT_QPLIB_SWQE_PAGE_SIZE_2M 9 |
219 | #define BNXT_QPLIB_SWQE_PAGE_SIZE_4M 10 |
220 | #define BNXT_QPLIB_SWQE_PAGE_SIZE_1G 18 |
221 | u8 levels; |
222 | #define PAGE_SHIFT_4K 12 |
223 | __le64 *pbl_ptr; |
224 | dma_addr_t pbl_dma_ptr; |
225 | u64 *page_list; |
226 | u16 page_list_len; |
227 | u64 va; |
228 | } frmr; |
229 | |
230 | /* Bind */ |
231 | struct { |
232 | u8 access_cntl; |
233 | #define BNXT_QPLIB_BIND_SWQE_ACCESS_LOCAL_WRITE BIT(0) |
234 | #define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_READ BIT(1) |
235 | #define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_WRITE BIT(2) |
236 | #define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_ATOMIC BIT(3) |
237 | #define BNXT_QPLIB_BIND_SWQE_ACCESS_WINDOW_BIND BIT(4) |
238 | bool zero_based; |
239 | u8 mw_type; |
240 | u32 parent_l_key; |
241 | u32 r_key; |
242 | u64 va; |
243 | u32 length; |
244 | } bind; |
245 | }; |
246 | }; |
247 | |
248 | struct bnxt_qplib_q { |
249 | struct bnxt_qplib_hwq hwq; |
250 | struct bnxt_qplib_swq *swq; |
251 | struct bnxt_qplib_db_info dbinfo; |
252 | struct bnxt_qplib_sg_info sg_info; |
253 | u32 max_wqe; |
254 | u16 wqe_size; |
255 | u16 q_full_delta; |
256 | u16 max_sge; |
257 | u32 psn; |
258 | bool condition; |
259 | bool single; |
260 | bool send_phantom; |
261 | u32 phantom_wqe_cnt; |
262 | u32 phantom_cqe_cnt; |
263 | u32 next_cq_cons; |
264 | bool flushed; |
265 | u32 swq_start; |
266 | u32 swq_last; |
267 | }; |
268 | |
269 | struct bnxt_qplib_qp { |
270 | struct bnxt_qplib_pd *pd; |
271 | struct bnxt_qplib_dpi *dpi; |
272 | struct bnxt_qplib_chip_ctx *cctx; |
273 | u64 qp_handle; |
274 | #define BNXT_QPLIB_QP_ID_INVALID 0xFFFFFFFF |
275 | u32 id; |
276 | u8 type; |
277 | u8 sig_type; |
278 | u8 wqe_mode; |
279 | u8 state; |
280 | u8 cur_qp_state; |
281 | u64 modify_flags; |
282 | u32 max_inline_data; |
283 | u32 mtu; |
284 | u8 path_mtu; |
285 | bool en_sqd_async_notify; |
286 | u16 pkey_index; |
287 | u32 qkey; |
288 | u32 dest_qp_id; |
289 | u8 access; |
290 | u8 timeout; |
291 | u8 retry_cnt; |
292 | u8 rnr_retry; |
293 | u64 wqe_cnt; |
294 | u32 min_rnr_timer; |
295 | u32 max_rd_atomic; |
296 | u32 max_dest_rd_atomic; |
297 | u32 dest_qpn; |
298 | u8 smac[6]; |
299 | u16 vlan_id; |
300 | u8 nw_type; |
301 | struct bnxt_qplib_ah ah; |
302 | |
303 | #define BTH_PSN_MASK ((1 << 24) - 1) |
304 | /* SQ */ |
305 | struct bnxt_qplib_q sq; |
306 | /* RQ */ |
307 | struct bnxt_qplib_q rq; |
308 | /* SRQ */ |
309 | struct bnxt_qplib_srq *srq; |
310 | /* CQ */ |
311 | struct bnxt_qplib_cq *scq; |
312 | struct bnxt_qplib_cq *rcq; |
313 | /* IRRQ and ORRQ */ |
314 | struct bnxt_qplib_hwq irrq; |
315 | struct bnxt_qplib_hwq orrq; |
316 | /* Header buffer for QP1 */ |
317 | int sq_hdr_buf_size; |
318 | int rq_hdr_buf_size; |
319 | /* |
320 | * Buffer space for ETH(14), IP or GRH(40), UDP header(8) |
321 | * and ib_bth + ib_deth (20). |
322 | * Max required is 82 when RoCE V2 is enabled |
323 | */ |
324 | #define BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2 86 |
325 | /* Ethernet header = 14 */ |
326 | /* ib_grh = 40 (provided by MAD) */ |
327 | /* ib_bth + ib_deth = 20 */ |
328 | /* MAD = 256 (provided by MAD) */ |
329 | /* iCRC = 4 */ |
330 | #define BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE 14 |
331 | #define BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2 512 |
332 | #define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 20 |
333 | #define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 40 |
334 | #define BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE 20 |
335 | void *sq_hdr_buf; |
336 | dma_addr_t sq_hdr_buf_map; |
337 | void *rq_hdr_buf; |
338 | dma_addr_t rq_hdr_buf_map; |
339 | struct list_head sq_flush; |
340 | struct list_head rq_flush; |
341 | u32 msn; |
342 | u32 msn_tbl_sz; |
343 | u16 dev_cap_flags; |
344 | }; |
345 | |
346 | #define BNXT_QPLIB_MAX_CQE_ENTRY_SIZE sizeof(struct cq_base) |
347 | |
348 | #define CQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_CQE_ENTRY_SIZE) |
349 | #define CQE_MAX_IDX_PER_PG (CQE_CNT_PER_PG - 1) |
350 | #define CQE_PG(x) (((x) & ~CQE_MAX_IDX_PER_PG) / CQE_CNT_PER_PG) |
351 | #define CQE_IDX(x) ((x) & CQE_MAX_IDX_PER_PG) |
352 | |
353 | #define ROCE_CQE_CMP_V 0 |
354 | #define CQE_CMP_VALID(hdr, pass) \ |
355 | (!!((hdr)->cqe_type_toggle & CQ_BASE_TOGGLE) == \ |
356 | !((pass) & BNXT_QPLIB_FLAG_EPOCH_CONS_MASK)) |
357 | |
358 | static inline u32 __bnxt_qplib_get_avail(struct bnxt_qplib_hwq *hwq) |
359 | { |
360 | int cons, prod, avail; |
361 | |
362 | cons = hwq->cons; |
363 | prod = hwq->prod; |
364 | avail = cons - prod; |
365 | if (cons <= prod) |
366 | avail += hwq->depth; |
367 | return avail; |
368 | } |
369 | |
370 | static inline bool bnxt_qplib_queue_full(struct bnxt_qplib_q *que, |
371 | u8 slots) |
372 | { |
373 | struct bnxt_qplib_hwq *hwq; |
374 | int avail; |
375 | |
376 | hwq = &que->hwq; |
377 | /* False full is possible, retrying post-send makes sense */ |
378 | avail = hwq->cons - hwq->prod; |
379 | if (hwq->cons <= hwq->prod) |
380 | avail += hwq->depth; |
381 | return avail <= slots; |
382 | } |
383 | |
384 | struct bnxt_qplib_cqe { |
385 | u8 status; |
386 | u8 type; |
387 | u8 opcode; |
388 | u32 length; |
389 | u16 cfa_meta; |
390 | u64 wr_id; |
391 | union { |
392 | __be32 immdata; |
393 | u32 invrkey; |
394 | }; |
395 | u64 qp_handle; |
396 | u64 mr_handle; |
397 | u16 flags; |
398 | u8 smac[6]; |
399 | u32 src_qp; |
400 | u16 raweth_qp1_flags; |
401 | u16 raweth_qp1_errors; |
402 | u16 raweth_qp1_cfa_code; |
403 | u32 raweth_qp1_flags2; |
404 | u32 raweth_qp1_metadata; |
405 | u8 raweth_qp1_payload_offset; |
406 | u16 pkey_index; |
407 | }; |
408 | |
409 | #define BNXT_QPLIB_QUEUE_START_PERIOD 0x01 |
410 | struct bnxt_qplib_cq { |
411 | struct bnxt_qplib_dpi *dpi; |
412 | struct bnxt_qplib_db_info dbinfo; |
413 | u32 max_wqe; |
414 | u32 id; |
415 | u16 count; |
416 | u16 period; |
417 | struct bnxt_qplib_hwq hwq; |
418 | struct bnxt_qplib_hwq resize_hwq; |
419 | u32 cnq_hw_ring_id; |
420 | struct bnxt_qplib_nq *nq; |
421 | bool resize_in_progress; |
422 | struct bnxt_qplib_sg_info sg_info; |
423 | u64 cq_handle; |
424 | u8 toggle; |
425 | |
426 | #define CQ_RESIZE_WAIT_TIME_MS 500 |
427 | unsigned long flags; |
428 | #define CQ_FLAGS_RESIZE_IN_PROG 1 |
429 | wait_queue_head_t waitq; |
430 | struct list_head sqf_head, rqf_head; |
431 | atomic_t arm_state; |
432 | spinlock_t compl_lock; /* synch CQ handlers */ |
433 | /* Locking Notes: |
434 | * QP can move to error state from modify_qp, async error event or error |
435 | * CQE as part of poll_cq. When QP is moved to error state, it gets added |
436 | * to two flush lists, one each for SQ and RQ. |
437 | * Each flush list is protected by qplib_cq->flush_lock. Both scq and rcq |
438 | * flush_locks should be acquired when QP is moved to error. The control path |
439 | * operations(modify_qp and async error events) are synchronized with poll_cq |
440 | * using upper level CQ locks (bnxt_re_cq->cq_lock) of both SCQ and RCQ. |
441 | * The qplib_cq->flush_lock is required to synchronize two instances of poll_cq |
442 | * of the same QP while manipulating the flush list. |
443 | */ |
444 | spinlock_t flush_lock; /* QP flush management */ |
445 | u16 cnq_events; |
446 | }; |
447 | |
448 | #define BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE sizeof(struct xrrq_irrq) |
449 | #define BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE sizeof(struct xrrq_orrq) |
450 | #define IRD_LIMIT_TO_IRRQ_SLOTS(x) (2 * (x) + 2) |
451 | #define IRRQ_SLOTS_TO_IRD_LIMIT(s) (((s) >> 1) - 1) |
452 | #define ORD_LIMIT_TO_ORRQ_SLOTS(x) ((x) + 1) |
453 | #define ORRQ_SLOTS_TO_ORD_LIMIT(s) ((s) - 1) |
454 | |
455 | #define BNXT_QPLIB_MAX_NQE_ENTRY_SIZE sizeof(struct nq_base) |
456 | |
457 | #define NQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_NQE_ENTRY_SIZE) |
458 | #define NQE_MAX_IDX_PER_PG (NQE_CNT_PER_PG - 1) |
459 | #define NQE_PG(x) (((x) & ~NQE_MAX_IDX_PER_PG) / NQE_CNT_PER_PG) |
460 | #define NQE_IDX(x) ((x) & NQE_MAX_IDX_PER_PG) |
461 | |
462 | #define NQE_CMP_VALID(hdr, pass) \ |
463 | (!!(le32_to_cpu((hdr)->info63_v[0]) & NQ_BASE_V) == \ |
464 | !((pass) & BNXT_QPLIB_FLAG_EPOCH_CONS_MASK)) |
465 | |
466 | #define BNXT_QPLIB_NQE_MAX_CNT (128 * 1024) |
467 | |
468 | #define NQ_CONS_PCI_BAR_REGION 2 |
469 | #define NQ_DB_KEY_CP (0x2 << CMPL_DOORBELL_KEY_SFT) |
470 | #define NQ_DB_IDX_VALID CMPL_DOORBELL_IDX_VALID |
471 | #define NQ_DB_IRQ_DIS CMPL_DOORBELL_MASK |
472 | #define NQ_DB_CP_FLAGS_REARM (NQ_DB_KEY_CP | \ |
473 | NQ_DB_IDX_VALID) |
474 | #define NQ_DB_CP_FLAGS (NQ_DB_KEY_CP | \ |
475 | NQ_DB_IDX_VALID | \ |
476 | NQ_DB_IRQ_DIS) |
477 | |
478 | struct bnxt_qplib_nq_db { |
479 | struct bnxt_qplib_reg_desc reg; |
480 | struct bnxt_qplib_db_info dbinfo; |
481 | }; |
482 | |
483 | typedef int (*cqn_handler_t)(struct bnxt_qplib_nq *nq, |
484 | struct bnxt_qplib_cq *cq); |
485 | typedef int (*srqn_handler_t)(struct bnxt_qplib_nq *nq, |
486 | struct bnxt_qplib_srq *srq, u8 event); |
487 | |
488 | struct bnxt_qplib_nq { |
489 | struct pci_dev *pdev; |
490 | struct bnxt_qplib_res *res; |
491 | char *name; |
492 | struct bnxt_qplib_hwq hwq; |
493 | struct bnxt_qplib_nq_db nq_db; |
494 | u16 ring_id; |
495 | int msix_vec; |
496 | cpumask_t mask; |
497 | struct tasklet_struct nq_tasklet; |
498 | bool requested; |
499 | int budget; |
500 | |
501 | cqn_handler_t cqn_handler; |
502 | srqn_handler_t srqn_handler; |
503 | struct workqueue_struct *cqn_wq; |
504 | }; |
505 | |
506 | struct bnxt_qplib_nq_work { |
507 | struct work_struct work; |
508 | struct bnxt_qplib_nq *nq; |
509 | struct bnxt_qplib_cq *cq; |
510 | }; |
511 | |
512 | void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill); |
513 | void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq); |
514 | int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx, |
515 | int msix_vector, bool need_init); |
516 | int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq, |
517 | int nq_idx, int msix_vector, int bar_reg_offset, |
518 | cqn_handler_t cqn_handler, |
519 | srqn_handler_t srq_handler); |
520 | int bnxt_qplib_create_srq(struct bnxt_qplib_res *res, |
521 | struct bnxt_qplib_srq *srq); |
522 | int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res, |
523 | struct bnxt_qplib_srq *srq); |
524 | int bnxt_qplib_query_srq(struct bnxt_qplib_res *res, |
525 | struct bnxt_qplib_srq *srq); |
526 | void bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res, |
527 | struct bnxt_qplib_srq *srq); |
528 | int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq, |
529 | struct bnxt_qplib_swqe *wqe); |
530 | int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); |
531 | int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); |
532 | int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); |
533 | int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); |
534 | int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); |
535 | void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp); |
536 | void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res, |
537 | struct bnxt_qplib_qp *qp); |
538 | void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp, |
539 | struct bnxt_qplib_sge *sge); |
540 | void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp, |
541 | struct bnxt_qplib_sge *sge); |
542 | u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp); |
543 | dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, |
544 | u32 index); |
545 | void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp); |
546 | int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, |
547 | struct bnxt_qplib_swqe *wqe); |
548 | void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp); |
549 | int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp, |
550 | struct bnxt_qplib_swqe *wqe); |
551 | int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq); |
552 | int bnxt_qplib_resize_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq, |
553 | int new_cqes); |
554 | void bnxt_qplib_resize_cq_complete(struct bnxt_qplib_res *res, |
555 | struct bnxt_qplib_cq *cq); |
556 | int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq); |
557 | int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe, |
558 | int num, struct bnxt_qplib_qp **qp); |
559 | bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq); |
560 | void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type); |
561 | void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq); |
562 | int bnxt_qplib_alloc_nq(struct bnxt_qplib_res *res, struct bnxt_qplib_nq *nq); |
563 | void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp); |
564 | void bnxt_qplib_acquire_cq_locks(struct bnxt_qplib_qp *qp, |
565 | unsigned long *flags); |
566 | void bnxt_qplib_release_cq_locks(struct bnxt_qplib_qp *qp, |
567 | unsigned long *flags); |
568 | int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq, |
569 | struct bnxt_qplib_cqe *cqe, |
570 | int num_cqes); |
571 | void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp); |
572 | void bnxt_re_synchronize_nq(struct bnxt_qplib_nq *nq); |
573 | |
574 | static inline void *bnxt_qplib_get_swqe(struct bnxt_qplib_q *que, u32 *swq_idx) |
575 | { |
576 | u32 idx; |
577 | |
578 | idx = que->swq_start; |
579 | if (swq_idx) |
580 | *swq_idx = idx; |
581 | return &que->swq[idx]; |
582 | } |
583 | |
584 | static inline void bnxt_qplib_swq_mod_start(struct bnxt_qplib_q *que, u32 idx) |
585 | { |
586 | que->swq_start = que->swq[idx].next_idx; |
587 | } |
588 | |
589 | static inline u32 bnxt_qplib_get_depth(struct bnxt_qplib_q *que) |
590 | { |
591 | return (que->wqe_size * que->max_wqe) / sizeof(struct sq_sge); |
592 | } |
593 | |
594 | static inline u32 bnxt_qplib_set_sq_size(struct bnxt_qplib_q *que, u8 wqe_mode) |
595 | { |
596 | return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? |
597 | que->max_wqe : bnxt_qplib_get_depth(que); |
598 | } |
599 | |
600 | static inline u32 bnxt_qplib_set_sq_max_slot(u8 wqe_mode) |
601 | { |
602 | return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? |
603 | sizeof(struct sq_send) / sizeof(struct sq_sge) : 1; |
604 | } |
605 | |
606 | static inline u32 bnxt_qplib_set_rq_max_slot(u32 wqe_size) |
607 | { |
608 | return (wqe_size / sizeof(struct sq_sge)); |
609 | } |
610 | |
611 | static inline u16 __xlate_qfd(u16 delta, u16 wqe_bytes) |
612 | { |
613 | /* For Cu/Wh delta = 128, stride = 16, wqe_bytes = 128 |
614 | * For Gen-p5 B/C mode delta = 0, stride = 16, wqe_bytes = 128. |
615 | * For Gen-p5 delta = 0, stride = 16, 32 <= wqe_bytes <= 512. |
616 | * when 8916 is disabled. |
617 | */ |
618 | return (delta * wqe_bytes) / sizeof(struct sq_sge); |
619 | } |
620 | |
621 | static inline u16 bnxt_qplib_calc_ilsize(struct bnxt_qplib_swqe *wqe, u16 max) |
622 | { |
623 | u16 size = 0; |
624 | int indx; |
625 | |
626 | for (indx = 0; indx < wqe->num_sge; indx++) |
627 | size += wqe->sg_list[indx].size; |
628 | if (size > max) |
629 | size = max; |
630 | |
631 | return size; |
632 | } |
633 | |
634 | /* MSN table update inlin */ |
635 | static inline __le64 bnxt_re_update_msn_tbl(u32 st_idx, u32 npsn, u32 start_psn) |
636 | { |
637 | return cpu_to_le64((((u64)(st_idx) << SQ_MSN_SEARCH_START_IDX_SFT) & |
638 | SQ_MSN_SEARCH_START_IDX_MASK) | |
639 | (((u64)(npsn) << SQ_MSN_SEARCH_NEXT_PSN_SFT) & |
640 | SQ_MSN_SEARCH_NEXT_PSN_MASK) | |
641 | (((start_psn) << SQ_MSN_SEARCH_START_PSN_SFT) & |
642 | SQ_MSN_SEARCH_START_PSN_MASK)); |
643 | } |
644 | #endif /* __BNXT_QPLIB_FP_H__ */ |
645 | |