1	/* QLogic qedr NIC Driver
2	* Copyright (c) 2015-2016 QLogic Corporation
3	*
4	* This software is available to you under a choice of one of two
5	* licenses. You may choose to be licensed under the terms of the GNU
6	* General Public License (GPL) Version 2, available from the file
7	* COPYING in the main directory of this source tree, or the
8	* OpenIB.org BSD license below:
9	*
10	* Redistribution and use in source and binary forms, with or
11	* without modification, are permitted provided that the following
12	* conditions are met:
13	*
14	* - Redistributions of source code must retain the above
15	* copyright notice, this list of conditions and the following
16	* disclaimer.
17	*
18	* - Redistributions in binary form must reproduce the above
19	* copyright notice, this list of conditions and the following
20	* disclaimer in the documentation and /or other materials
21	* provided with the distribution.
22	*
23	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30	* SOFTWARE.
31	*/
32	#include <linux/dma-mapping.h>
33	#include <linux/crc32.h>
34	#include <net/ip.h>
35	#include <net/ipv6.h>
36	#include <net/udp.h>
37	#include <linux/iommu.h>
38
39	#include <rdma/ib_verbs.h>
40	#include <rdma/ib_user_verbs.h>
41	#include <rdma/iw_cm.h>
42	#include <rdma/ib_umem.h>
43	#include <rdma/ib_addr.h>
44	#include <rdma/ib_cache.h>
45	#include <rdma/uverbs_ioctl.h>
46
47	#include <linux/qed/common_hsi.h>
48	#include "qedr_hsi_rdma.h"
49	#include <linux/qed/qed_if.h>
50	#include "qedr.h"
51	#include "verbs.h"
52	#include <rdma/qedr-abi.h>
53	#include "qedr_roce_cm.h"
54	#include "qedr_iw_cm.h"
55
56	#define QEDR_SRQ_WQE_ELEM_SIZE sizeof(union rdma_srq_elm)
57	#define RDMA_MAX_SGE_PER_SRQ (4)
58	#define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1)
59
60	#define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
61
62	enum {
63	QEDR_USER_MMAP_IO_WC = 0,
64	QEDR_USER_MMAP_PHYS_PAGE,
65	};
66
67	static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
68	size_t len)
69	{
70	size_t min_len = min_t(size_t, len, udata->outlen);
71
72	return ib_copy_to_udata(udata, src, len: min_len);
73	}
74
75	int qedr_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey)
76	{
77	if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
78	return -EINVAL;
79
80	*pkey = QEDR_ROCE_PKEY_DEFAULT;
81	return 0;
82	}
83
84	int qedr_iw_query_gid(struct ib_device *ibdev, u32 port,
85	int index, union ib_gid *sgid)
86	{
87	struct qedr_dev *dev = get_qedr_dev(ibdev);
88
89	memset(sgid->raw, 0, sizeof(sgid->raw));
90	ether_addr_copy(dst: sgid->raw, src: dev->ndev->dev_addr);
91
92	DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
93	sgid->global.interface_id, sgid->global.subnet_prefix);
94
95	return 0;
96	}
97
98	int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
99	{
100	struct qedr_dev *dev = get_qedr_dev(ibdev: ibsrq->device);
101	struct qedr_device_attr *qattr = &dev->attr;
102	struct qedr_srq *srq = get_qedr_srq(ibsrq);
103
104	srq_attr->srq_limit = srq->srq_limit;
105	srq_attr->max_wr = qattr->max_srq_wr;
106	srq_attr->max_sge = qattr->max_sge;
107
108	return 0;
109	}
110
111	int qedr_query_device(struct ib_device *ibdev,
112	struct ib_device_attr *attr, struct ib_udata *udata)
113	{
114	struct qedr_dev *dev = get_qedr_dev(ibdev);
115	struct qedr_device_attr *qattr = &dev->attr;
116
117	if (!dev->rdma_ctx) {
118	DP_ERR(dev,
119	"qedr_query_device called with invalid params rdma_ctx=%p\n",
120	dev->rdma_ctx);
121	return -EINVAL;
122	}
123
124	memset(attr, 0, sizeof(*attr));
125
126	attr->fw_ver = qattr->fw_ver;
127	attr->sys_image_guid = qattr->sys_image_guid;
128	attr->max_mr_size = qattr->max_mr_size;
129	attr->page_size_cap = qattr->page_size_caps;
130	attr->vendor_id = qattr->vendor_id;
131	attr->vendor_part_id = qattr->vendor_part_id;
132	attr->hw_ver = qattr->hw_ver;
133	attr->max_qp = qattr->max_qp;
134	attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
135	attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
136	IB_DEVICE_RC_RNR_NAK_GEN |
137	IB_DEVICE_MEM_MGT_EXTENSIONS;
138	attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
139
140	if (!rdma_protocol_iwarp(device: &dev->ibdev, port_num: 1))
141	attr->device_cap_flags |= IB_DEVICE_XRC;
142	attr->max_send_sge = qattr->max_sge;
143	attr->max_recv_sge = qattr->max_sge;
144	attr->max_sge_rd = qattr->max_sge;
145	attr->max_cq = qattr->max_cq;
146	attr->max_cqe = qattr->max_cqe;
147	attr->max_mr = qattr->max_mr;
148	attr->max_mw = qattr->max_mw;
149	attr->max_pd = qattr->max_pd;
150	attr->atomic_cap = dev->atomic_cap;
151	attr->max_qp_init_rd_atom =
152	1 << (fls(x: qattr->max_qp_req_rd_atomic_resc) - 1);
153	attr->max_qp_rd_atom =
154	min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
155	attr->max_qp_init_rd_atom);
156
157	attr->max_srq = qattr->max_srq;
158	attr->max_srq_sge = qattr->max_srq_sge;
159	attr->max_srq_wr = qattr->max_srq_wr;
160
161	attr->local_ca_ack_delay = qattr->dev_ack_delay;
162	attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
163	attr->max_pkeys = qattr->max_pkey;
164	attr->max_ah = qattr->max_ah;
165
166	return 0;
167	}
168
169	static inline void get_link_speed_and_width(int speed, u16 *ib_speed,
170	u8 *ib_width)
171	{
172	switch (speed) {
173	case 1000:
174	*ib_speed = IB_SPEED_SDR;
175	*ib_width = IB_WIDTH_1X;
176	break;
177	case 10000:
178	*ib_speed = IB_SPEED_QDR;
179	*ib_width = IB_WIDTH_1X;
180	break;
181
182	case 20000:
183	*ib_speed = IB_SPEED_DDR;
184	*ib_width = IB_WIDTH_4X;
185	break;
186
187	case 25000:
188	*ib_speed = IB_SPEED_EDR;
189	*ib_width = IB_WIDTH_1X;
190	break;
191
192	case 40000:
193	*ib_speed = IB_SPEED_QDR;
194	*ib_width = IB_WIDTH_4X;
195	break;
196
197	case 50000:
198	*ib_speed = IB_SPEED_HDR;
199	*ib_width = IB_WIDTH_1X;
200	break;
201
202	case 100000:
203	*ib_speed = IB_SPEED_EDR;
204	*ib_width = IB_WIDTH_4X;
205	break;
206
207	default:
208	/* Unsupported */
209	*ib_speed = IB_SPEED_SDR;
210	*ib_width = IB_WIDTH_1X;
211	}
212	}
213
214	int qedr_query_port(struct ib_device *ibdev, u32 port,
215	struct ib_port_attr *attr)
216	{
217	struct qedr_dev *dev;
218	struct qed_rdma_port *rdma_port;
219
220	dev = get_qedr_dev(ibdev);
221
222	if (!dev->rdma_ctx) {
223	DP_ERR(dev, "rdma_ctx is NULL\n");
224	return -EINVAL;
225	}
226
227	rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
228
229	/* *attr being zeroed by the caller, avoid zeroing it here */
230	if (rdma_port->port_state == QED_RDMA_PORT_UP) {
231	attr->state = IB_PORT_ACTIVE;
232	attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
233	} else {
234	attr->state = IB_PORT_DOWN;
235	attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
236	}
237	attr->max_mtu = IB_MTU_4096;
238	attr->lid = 0;
239	attr->lmc = 0;
240	attr->sm_lid = 0;
241	attr->sm_sl = 0;
242	attr->ip_gids = true;
243	if (rdma_protocol_iwarp(device: &dev->ibdev, port_num: 1)) {
244	attr->active_mtu = iboe_get_mtu(mtu: dev->iwarp_max_mtu);
245	attr->gid_tbl_len = 1;
246	} else {
247	attr->active_mtu = iboe_get_mtu(mtu: dev->ndev->mtu);
248	attr->gid_tbl_len = QEDR_MAX_SGID;
249	attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
250	}
251	attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
252	attr->qkey_viol_cntr = 0;
253	get_link_speed_and_width(speed: rdma_port->link_speed,
254	ib_speed: &attr->active_speed, ib_width: &attr->active_width);
255	attr->max_msg_sz = rdma_port->max_msg_size;
256	attr->max_vl_num = 4;
257
258	return 0;
259	}
260
261	int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
262	{
263	struct ib_device *ibdev = uctx->device;
264	int rc;
265	struct qedr_ucontext *ctx = get_qedr_ucontext(ibucontext: uctx);
266	struct qedr_alloc_ucontext_resp uresp = {};
267	struct qedr_alloc_ucontext_req ureq = {};
268	struct qedr_dev *dev = get_qedr_dev(ibdev);
269	struct qed_rdma_add_user_out_params oparams;
270	struct qedr_user_mmap_entry *entry;
271
272	if (!udata)
273	return -EFAULT;
274
275	if (udata->inlen) {
276	rc = ib_copy_from_udata(dest: &ureq, udata,
277	min(sizeof(ureq), udata->inlen));
278	if (rc) {
279	DP_ERR(dev, "Problem copying data from user space\n");
280	return -EFAULT;
281	}
282	ctx->edpm_mode = !!(ureq.context_flags &
283	QEDR_ALLOC_UCTX_EDPM_MODE);
284	ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC);
285	}
286
287	rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
288	if (rc) {
289	DP_ERR(dev,
290	"failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
291	rc);
292	return rc;
293	}
294
295	ctx->dpi = oparams.dpi;
296	ctx->dpi_addr = oparams.dpi_addr;
297	ctx->dpi_phys_addr = oparams.dpi_phys_addr;
298	ctx->dpi_size = oparams.dpi_size;
299	entry = kzalloc(size: sizeof(*entry), GFP_KERNEL);
300	if (!entry) {
301	rc = -ENOMEM;
302	goto err;
303	}
304
305	entry->io_address = ctx->dpi_phys_addr;
306	entry->length = ctx->dpi_size;
307	entry->mmap_flag = QEDR_USER_MMAP_IO_WC;
308	entry->dpi = ctx->dpi;
309	entry->dev = dev;
310	rc = rdma_user_mmap_entry_insert(ucontext: uctx, entry: &entry->rdma_entry,
311	length: ctx->dpi_size);
312	if (rc) {
313	kfree(objp: entry);
314	goto err;
315	}
316	ctx->db_mmap_entry = &entry->rdma_entry;
317
318	if (!dev->user_dpm_enabled)
319	uresp.dpm_flags = 0;
320	else if (rdma_protocol_iwarp(device: &dev->ibdev, port_num: 1))
321	uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY;
322	else
323	uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED |
324	QEDR_DPM_TYPE_ROCE_LEGACY |
325	QEDR_DPM_TYPE_ROCE_EDPM_MODE;
326
327	if (ureq.context_flags & QEDR_SUPPORT_DPM_SIZES) {
328	uresp.dpm_flags |= QEDR_DPM_SIZES_SET;
329	uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE;
330	uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE;
331	uresp.edpm_limit_size = QEDR_EDPM_MAX_SIZE;
332	}
333
334	uresp.wids_enabled = 1;
335	uresp.wid_count = oparams.wid_count;
336	uresp.db_pa = rdma_user_mmap_get_offset(entry: ctx->db_mmap_entry);
337	uresp.db_size = ctx->dpi_size;
338	uresp.max_send_wr = dev->attr.max_sqe;
339	uresp.max_recv_wr = dev->attr.max_rqe;
340	uresp.max_srq_wr = dev->attr.max_srq_wr;
341	uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
342	uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
343	uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
344	uresp.max_cqes = QEDR_MAX_CQES;
345
346	rc = qedr_ib_copy_to_udata(udata, src: &uresp, len: sizeof(uresp));
347	if (rc)
348	goto err;
349
350	ctx->dev = dev;
351
352	DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
353	&ctx->ibucontext);
354	return 0;
355
356	err:
357	if (!ctx->db_mmap_entry)
358	dev->ops->rdma_remove_user(dev->rdma_ctx, ctx->dpi);
359	else
360	rdma_user_mmap_entry_remove(entry: ctx->db_mmap_entry);
361
362	return rc;
363	}
364
365	void qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
366	{
367	struct qedr_ucontext *uctx = get_qedr_ucontext(ibucontext: ibctx);
368
369	DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
370	uctx);
371
372	rdma_user_mmap_entry_remove(entry: uctx->db_mmap_entry);
373	}
374
375	void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
376	{
377	struct qedr_user_mmap_entry *entry = get_qedr_mmap_entry(rdma_entry);
378	struct qedr_dev *dev = entry->dev;
379
380	if (entry->mmap_flag == QEDR_USER_MMAP_PHYS_PAGE)
381	free_page((unsigned long)entry->address);
382	else if (entry->mmap_flag == QEDR_USER_MMAP_IO_WC)
383	dev->ops->rdma_remove_user(dev->rdma_ctx, entry->dpi);
384
385	kfree(objp: entry);
386	}
387
388	int qedr_mmap(struct ib_ucontext *ucontext, struct vm_area_struct *vma)
389	{
390	struct ib_device *dev = ucontext->device;
391	size_t length = vma->vm_end - vma->vm_start;
392	struct rdma_user_mmap_entry *rdma_entry;
393	struct qedr_user_mmap_entry *entry;
394	int rc = 0;
395	u64 pfn;
396
397	ibdev_dbg(dev,
398	"start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n",
399	vma->vm_start, vma->vm_end, length, vma->vm_pgoff);
400
401	rdma_entry = rdma_user_mmap_entry_get(ucontext, vma);
402	if (!rdma_entry) {
403	ibdev_dbg(dev, "pgoff[%#lx] does not have valid entry\n",
404	vma->vm_pgoff);
405	return -EINVAL;
406	}
407	entry = get_qedr_mmap_entry(rdma_entry);
408	ibdev_dbg(dev,
409	"Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n",
410	entry->io_address, length, entry->mmap_flag);
411
412	switch (entry->mmap_flag) {
413	case QEDR_USER_MMAP_IO_WC:
414	pfn = entry->io_address >> PAGE_SHIFT;
415	rc = rdma_user_mmap_io(ucontext, vma, pfn, size: length,
416	pgprot_writecombine(prot: vma->vm_page_prot),
417	entry: rdma_entry);
418	break;
419	case QEDR_USER_MMAP_PHYS_PAGE:
420	rc = vm_insert_page(vma, addr: vma->vm_start,
421	virt_to_page(entry->address));
422	break;
423	default:
424	rc = -EINVAL;
425	}
426
427	if (rc)
428	ibdev_dbg(dev,
429	"Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",
430	entry->io_address, length, entry->mmap_flag, rc);
431
432	rdma_user_mmap_entry_put(entry: rdma_entry);
433	return rc;
434	}
435
436	int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
437	{
438	struct ib_device *ibdev = ibpd->device;
439	struct qedr_dev *dev = get_qedr_dev(ibdev);
440	struct qedr_pd *pd = get_qedr_pd(ibpd);
441	u16 pd_id;
442	int rc;
443
444	DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
445	udata ? "User Lib" : "Kernel");
446
447	if (!dev->rdma_ctx) {
448	DP_ERR(dev, "invalid RDMA context\n");
449	return -EINVAL;
450	}
451
452	rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
453	if (rc)
454	return rc;
455
456	pd->pd_id = pd_id;
457
458	if (udata) {
459	struct qedr_alloc_pd_uresp uresp = {
460	.pd_id = pd_id,
461	};
462	struct qedr_ucontext *context = rdma_udata_to_drv_context(
463	udata, struct qedr_ucontext, ibucontext);
464
465	rc = qedr_ib_copy_to_udata(udata, src: &uresp, len: sizeof(uresp));
466	if (rc) {
467	DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
468	dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
469	return rc;
470	}
471
472	pd->uctx = context;
473	pd->uctx->pd = pd;
474	}
475
476	return 0;
477	}
478
479	int qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
480	{
481	struct qedr_dev *dev = get_qedr_dev(ibdev: ibpd->device);
482	struct qedr_pd *pd = get_qedr_pd(ibpd);
483
484	DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
485	dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
486	return 0;
487	}
488
489
490	int qedr_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
491	{
492	struct qedr_dev *dev = get_qedr_dev(ibdev: ibxrcd->device);
493	struct qedr_xrcd *xrcd = get_qedr_xrcd(ibxrcd);
494
495	return dev->ops->rdma_alloc_xrcd(dev->rdma_ctx, &xrcd->xrcd_id);
496	}
497
498	int qedr_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
499	{
500	struct qedr_dev *dev = get_qedr_dev(ibdev: ibxrcd->device);
501	u16 xrcd_id = get_qedr_xrcd(ibxrcd)->xrcd_id;
502
503	dev->ops->rdma_dealloc_xrcd(dev->rdma_ctx, xrcd_id);
504	return 0;
505	}
506	static void qedr_free_pbl(struct qedr_dev *dev,
507	struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
508	{
509	struct pci_dev *pdev = dev->pdev;
510	int i;
511
512	for (i = 0; i < pbl_info->num_pbls; i++) {
513	if (!pbl[i].va)
514	continue;
515	dma_free_coherent(dev: &pdev->dev, size: pbl_info->pbl_size,
516	cpu_addr: pbl[i].va, dma_handle: pbl[i].pa);
517	}
518
519	kfree(objp: pbl);
520	}
521
522	#define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
523	#define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
524
525	#define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
526	#define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
527	#define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
528
529	static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
530	struct qedr_pbl_info *pbl_info,
531	gfp_t flags)
532	{
533	struct pci_dev *pdev = dev->pdev;
534	struct qedr_pbl *pbl_table;
535	dma_addr_t *pbl_main_tbl;
536	dma_addr_t pa;
537	void *va;
538	int i;
539
540	pbl_table = kcalloc(n: pbl_info->num_pbls, size: sizeof(*pbl_table), flags);
541	if (!pbl_table)
542	return ERR_PTR(error: -ENOMEM);
543
544	for (i = 0; i < pbl_info->num_pbls; i++) {
545	va = dma_alloc_coherent(dev: &pdev->dev, size: pbl_info->pbl_size, dma_handle: &pa,
546	gfp: flags);
547	if (!va)
548	goto err;
549
550	pbl_table[i].va = va;
551	pbl_table[i].pa = pa;
552	}
553
554	/* Two-Layer PBLs, if we have more than one pbl we need to initialize
555	* the first one with physical pointers to all of the rest
556	*/
557	pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
558	for (i = 0; i < pbl_info->num_pbls - 1; i++)
559	pbl_main_tbl[i] = pbl_table[i + 1].pa;
560
561	return pbl_table;
562
563	err:
564	for (i--; i >= 0; i--)
565	dma_free_coherent(dev: &pdev->dev, size: pbl_info->pbl_size,
566	cpu_addr: pbl_table[i].va, dma_handle: pbl_table[i].pa);
567
568	qedr_free_pbl(dev, pbl_info, pbl: pbl_table);
569
570	return ERR_PTR(error: -ENOMEM);
571	}
572
573	static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
574	struct qedr_pbl_info *pbl_info,
575	u32 num_pbes, int two_layer_capable)
576	{
577	u32 pbl_capacity;
578	u32 pbl_size;
579	u32 num_pbls;
580
581	if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
582	if (num_pbes > MAX_PBES_TWO_LAYER) {
583	DP_ERR(dev, "prepare pbl table: too many pages %d\n",
584	num_pbes);
585	return -EINVAL;
586	}
587
588	/* calculate required pbl page size */
589	pbl_size = MIN_FW_PBL_PAGE_SIZE;
590	pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
591	NUM_PBES_ON_PAGE(pbl_size);
592
593	while (pbl_capacity < num_pbes) {
594	pbl_size *= 2;
595	pbl_capacity = pbl_size / sizeof(u64);
596	pbl_capacity = pbl_capacity * pbl_capacity;
597	}
598
599	num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
600	num_pbls++; /* One for the layer0 ( points to the pbls) */
601	pbl_info->two_layered = true;
602	} else {
603	/* One layered PBL */
604	num_pbls = 1;
605	pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
606	roundup_pow_of_two((num_pbes * sizeof(u64))));
607	pbl_info->two_layered = false;
608	}
609
610	pbl_info->num_pbls = num_pbls;
611	pbl_info->pbl_size = pbl_size;
612	pbl_info->num_pbes = num_pbes;
613
614	DP_DEBUG(dev, QEDR_MSG_MR,
615	"prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
616	pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
617
618	return 0;
619	}
620
621	static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
622	struct qedr_pbl *pbl,
623	struct qedr_pbl_info *pbl_info, u32 pg_shift)
624	{
625	int pbe_cnt, total_num_pbes = 0;
626	struct qedr_pbl *pbl_tbl;
627	struct ib_block_iter biter;
628	struct regpair *pbe;
629
630	if (!pbl_info->num_pbes)
631	return;
632
633	/* If we have a two layered pbl, the first pbl points to the rest
634	* of the pbls and the first entry lays on the second pbl in the table
635	*/
636	if (pbl_info->two_layered)
637	pbl_tbl = &pbl[1];
638	else
639	pbl_tbl = pbl;
640
641	pbe = (struct regpair *)pbl_tbl->va;
642	if (!pbe) {
643	DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
644	return;
645	}
646
647	pbe_cnt = 0;
648
649	rdma_umem_for_each_dma_block (umem, &biter, BIT(pg_shift)) {
650	u64 pg_addr = rdma_block_iter_dma_address(biter: &biter);
651
652	pbe->lo = cpu_to_le32(pg_addr);
653	pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
654
655	pbe_cnt++;
656	total_num_pbes++;
657	pbe++;
658
659	if (total_num_pbes == pbl_info->num_pbes)
660	return;
661
662	/* If the given pbl is full storing the pbes, move to next pbl.
663	*/
664	if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
665	pbl_tbl++;
666	pbe = (struct regpair *)pbl_tbl->va;
667	pbe_cnt = 0;
668	}
669	}
670	}
671
672	static int qedr_db_recovery_add(struct qedr_dev *dev,
673	void __iomem *db_addr,
674	void *db_data,
675	enum qed_db_rec_width db_width,
676	enum qed_db_rec_space db_space)
677	{
678	if (!db_data) {
679	DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
680	return 0;
681	}
682
683	return dev->ops->common->db_recovery_add(dev->cdev, db_addr, db_data,
684	db_width, db_space);
685	}
686
687	static void qedr_db_recovery_del(struct qedr_dev *dev,
688	void __iomem *db_addr,
689	void *db_data)
690	{
691	if (!db_data) {
692	DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
693	return;
694	}
695
696	/* Ignore return code as there is not much we can do about it. Error
697	* log will be printed inside.
698	*/
699	dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data);
700	}
701
702	static int qedr_copy_cq_uresp(struct qedr_dev *dev,
703	struct qedr_cq *cq, struct ib_udata *udata,
704	u32 db_offset)
705	{
706	struct qedr_create_cq_uresp uresp;
707	int rc;
708
709	memset(&uresp, 0, sizeof(uresp));
710
711	uresp.db_offset = db_offset;
712	uresp.icid = cq->icid;
713	if (cq->q.db_mmap_entry)
714	uresp.db_rec_addr =
715	rdma_user_mmap_get_offset(entry: cq->q.db_mmap_entry);
716
717	rc = qedr_ib_copy_to_udata(udata, src: &uresp, len: sizeof(uresp));
718	if (rc)
719	DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
720
721	return rc;
722	}
723
724	static void consume_cqe(struct qedr_cq *cq)
725	{
726	if (cq->latest_cqe == cq->toggle_cqe)
727	cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
728
729	cq->latest_cqe = qed_chain_consume(p_chain: &cq->pbl);
730	}
731
732	static inline int qedr_align_cq_entries(int entries)
733	{
734	u64 size, aligned_size;
735
736	/* We allocate an extra entry that we don't report to the FW. */
737	size = (entries + 1) * QEDR_CQE_SIZE;
738	aligned_size = ALIGN(size, PAGE_SIZE);
739
740	return aligned_size / QEDR_CQE_SIZE;
741	}
742
743	static int qedr_init_user_db_rec(struct ib_udata *udata,
744	struct qedr_dev *dev, struct qedr_userq *q,
745	bool requires_db_rec)
746	{
747	struct qedr_ucontext *uctx =
748	rdma_udata_to_drv_context(udata, struct qedr_ucontext,
749	ibucontext);
750	struct qedr_user_mmap_entry *entry;
751	int rc;
752
753	/* Aborting for non doorbell userqueue (SRQ) or non-supporting lib */
754	if (requires_db_rec == 0 || !uctx->db_rec)
755	return 0;
756
757	/* Allocate a page for doorbell recovery, add to mmap */
758	q->db_rec_data = (void *)get_zeroed_page(GFP_USER);
759	if (!q->db_rec_data) {
760	DP_ERR(dev, "get_zeroed_page failed\n");
761	return -ENOMEM;
762	}
763
764	entry = kzalloc(size: sizeof(*entry), GFP_KERNEL);
765	if (!entry)
766	goto err_free_db_data;
767
768	entry->address = q->db_rec_data;
769	entry->length = PAGE_SIZE;
770	entry->mmap_flag = QEDR_USER_MMAP_PHYS_PAGE;
771	rc = rdma_user_mmap_entry_insert(ucontext: &uctx->ibucontext,
772	entry: &entry->rdma_entry,
773	PAGE_SIZE);
774	if (rc)
775	goto err_free_entry;
776
777	q->db_mmap_entry = &entry->rdma_entry;
778
779	return 0;
780
781	err_free_entry:
782	kfree(objp: entry);
783
784	err_free_db_data:
785	free_page((unsigned long)q->db_rec_data);
786	q->db_rec_data = NULL;
787	return -ENOMEM;
788	}
789
790	static inline int qedr_init_user_queue(struct ib_udata *udata,
791	struct qedr_dev *dev,
792	struct qedr_userq *q, u64 buf_addr,
793	size_t buf_len, bool requires_db_rec,
794	int access,
795	int alloc_and_init)
796	{
797	u32 fw_pages;
798	int rc;
799
800	q->buf_addr = buf_addr;
801	q->buf_len = buf_len;
802	q->umem = ib_umem_get(device: &dev->ibdev, addr: q->buf_addr, size: q->buf_len, access);
803	if (IS_ERR(ptr: q->umem)) {
804	DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
805	PTR_ERR(q->umem));
806	return PTR_ERR(ptr: q->umem);
807	}
808
809	fw_pages = ib_umem_num_dma_blocks(umem: q->umem, pgsz: 1 << FW_PAGE_SHIFT);
810	rc = qedr_prepare_pbl_tbl(dev, pbl_info: &q->pbl_info, num_pbes: fw_pages, two_layer_capable: 0);
811	if (rc)
812	goto err0;
813
814	if (alloc_and_init) {
815	q->pbl_tbl = qedr_alloc_pbl_tbl(dev, pbl_info: &q->pbl_info, GFP_KERNEL);
816	if (IS_ERR(ptr: q->pbl_tbl)) {
817	rc = PTR_ERR(ptr: q->pbl_tbl);
818	goto err0;
819	}
820	qedr_populate_pbls(dev, umem: q->umem, pbl: q->pbl_tbl, pbl_info: &q->pbl_info,
821	FW_PAGE_SHIFT);
822	} else {
823	q->pbl_tbl = kzalloc(size: sizeof(*q->pbl_tbl), GFP_KERNEL);
824	if (!q->pbl_tbl) {
825	rc = -ENOMEM;
826	goto err0;
827	}
828	}
829
830	/* mmap the user address used to store doorbell data for recovery */
831	return qedr_init_user_db_rec(udata, dev, q, requires_db_rec);
832
833	err0:
834	ib_umem_release(umem: q->umem);
835	q->umem = NULL;
836
837	return rc;
838	}
839
840	static inline void qedr_init_cq_params(struct qedr_cq *cq,
841	struct qedr_ucontext *ctx,
842	struct qedr_dev *dev, int vector,
843	int chain_entries, int page_cnt,
844	u64 pbl_ptr,
845	struct qed_rdma_create_cq_in_params
846	*params)
847	{
848	memset(params, 0, sizeof(*params));
849	params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
850	params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
851	params->cnq_id = vector;
852	params->cq_size = chain_entries - 1;
853	params->dpi = (ctx) ? ctx->dpi : dev->dpi;
854	params->pbl_num_pages = page_cnt;
855	params->pbl_ptr = pbl_ptr;
856	params->pbl_two_level = 0;
857	}
858
859	static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
860	{
861	cq->db.data.agg_flags = flags;
862	cq->db.data.value = cpu_to_le32(cons);
863	writeq(val: cq->db.raw, addr: cq->db_addr);
864	}
865
866	int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
867	{
868	struct qedr_cq *cq = get_qedr_cq(ibcq);
869	unsigned long sflags;
870	struct qedr_dev *dev;
871
872	dev = get_qedr_dev(ibdev: ibcq->device);
873
874	if (cq->destroyed) {
875	DP_ERR(dev,
876	"warning: arm was invoked after destroy for cq %p (icid=%d)\n",
877	cq, cq->icid);
878	return -EINVAL;
879	}
880
881
882	if (cq->cq_type == QEDR_CQ_TYPE_GSI)
883	return 0;
884
885	spin_lock_irqsave(&cq->cq_lock, sflags);
886
887	cq->arm_flags = 0;
888
889	if (flags & IB_CQ_SOLICITED)
890	cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
891
892	if (flags & IB_CQ_NEXT_COMP)
893	cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
894
895	doorbell_cq(cq, cons: cq->cq_cons - 1, flags: cq->arm_flags);
896
897	spin_unlock_irqrestore(lock: &cq->cq_lock, flags: sflags);
898
899	return 0;
900	}
901
902	int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
903	struct ib_udata *udata)
904	{
905	struct ib_device *ibdev = ibcq->device;
906	struct qedr_ucontext *ctx = rdma_udata_to_drv_context(
907	udata, struct qedr_ucontext, ibucontext);
908	struct qed_rdma_destroy_cq_out_params destroy_oparams;
909	struct qed_rdma_destroy_cq_in_params destroy_iparams;
910	struct qed_chain_init_params chain_params = {
911	.mode = QED_CHAIN_MODE_PBL,
912	.intended_use = QED_CHAIN_USE_TO_CONSUME,
913	.cnt_type = QED_CHAIN_CNT_TYPE_U32,
914	.elem_size = sizeof(union rdma_cqe),
915	};
916	struct qedr_dev *dev = get_qedr_dev(ibdev);
917	struct qed_rdma_create_cq_in_params params;
918	struct qedr_create_cq_ureq ureq = {};
919	int vector = attr->comp_vector;
920	int entries = attr->cqe;
921	struct qedr_cq *cq = get_qedr_cq(ibcq);
922	int chain_entries;
923	u32 db_offset;
924	int page_cnt;
925	u64 pbl_ptr;
926	u16 icid;
927	int rc;
928
929	DP_DEBUG(dev, QEDR_MSG_INIT,
930	"create_cq: called from %s. entries=%d, vector=%d\n",
931	udata ? "User Lib" : "Kernel", entries, vector);
932
933	if (attr->flags)
934	return -EOPNOTSUPP;
935
936	if (entries > QEDR_MAX_CQES) {
937	DP_ERR(dev,
938	"create cq: the number of entries %d is too high. Must be equal or below %d.\n",
939	entries, QEDR_MAX_CQES);
940	return -EINVAL;
941	}
942
943	chain_entries = qedr_align_cq_entries(entries);
944	chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
945	chain_params.num_elems = chain_entries;
946
947	/* calc db offset. user will add DPI base, kernel will add db addr */
948	db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
949
950	if (udata) {
951	if (ib_copy_from_udata(dest: &ureq, udata, min(sizeof(ureq),
952	udata->inlen))) {
953	DP_ERR(dev,
954	"create cq: problem copying data from user space\n");
955	goto err0;
956	}
957
958	if (!ureq.len) {
959	DP_ERR(dev,
960	"create cq: cannot create a cq with 0 entries\n");
961	goto err0;
962	}
963
964	cq->cq_type = QEDR_CQ_TYPE_USER;
965
966	rc = qedr_init_user_queue(udata, dev, q: &cq->q, buf_addr: ureq.addr,
967	buf_len: ureq.len, requires_db_rec: true, access: IB_ACCESS_LOCAL_WRITE,
968	alloc_and_init: 1);
969	if (rc)
970	goto err0;
971
972	pbl_ptr = cq->q.pbl_tbl->pa;
973	page_cnt = cq->q.pbl_info.num_pbes;
974
975	cq->ibcq.cqe = chain_entries;
976	cq->q.db_addr = ctx->dpi_addr + db_offset;
977	} else {
978	cq->cq_type = QEDR_CQ_TYPE_KERNEL;
979
980	rc = dev->ops->common->chain_alloc(dev->cdev, &cq->pbl,
981	&chain_params);
982	if (rc)
983	goto err0;
984
985	page_cnt = qed_chain_get_page_cnt(chain: &cq->pbl);
986	pbl_ptr = qed_chain_get_pbl_phys(chain: &cq->pbl);
987	cq->ibcq.cqe = cq->pbl.capacity;
988	}
989
990	qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
991	pbl_ptr, params: &params);
992
993	rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
994	if (rc)
995	goto err1;
996
997	cq->icid = icid;
998	cq->sig = QEDR_CQ_MAGIC_NUMBER;
999	spin_lock_init(&cq->cq_lock);
1000
1001	if (udata) {
1002	rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset);
1003	if (rc)
1004	goto err2;
1005
1006	rc = qedr_db_recovery_add(dev, db_addr: cq->q.db_addr,
1007	db_data: &cq->q.db_rec_data->db_data,
1008	db_width: DB_REC_WIDTH_64B,
1009	db_space: DB_REC_USER);
1010	if (rc)
1011	goto err2;
1012
1013	} else {
1014	/* Generate doorbell address. */
1015	cq->db.data.icid = cq->icid;
1016	cq->db_addr = dev->db_addr + db_offset;
1017	cq->db.data.params = DB_AGG_CMD_MAX <<
1018	RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
1019
1020	/* point to the very last element, passing it we will toggle */
1021	cq->toggle_cqe = qed_chain_get_last_elem(p_chain: &cq->pbl);
1022	cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
1023	cq->latest_cqe = NULL;
1024	consume_cqe(cq);
1025	cq->cq_cons = qed_chain_get_cons_idx_u32(chain: &cq->pbl);
1026
1027	rc = qedr_db_recovery_add(dev, db_addr: cq->db_addr, db_data: &cq->db.data,
1028	db_width: DB_REC_WIDTH_64B, db_space: DB_REC_KERNEL);
1029	if (rc)
1030	goto err2;
1031	}
1032
1033	DP_DEBUG(dev, QEDR_MSG_CQ,
1034	"create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
1035	cq->icid, cq, params.cq_size);
1036
1037	return 0;
1038
1039	err2:
1040	destroy_iparams.icid = cq->icid;
1041	dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
1042	&destroy_oparams);
1043	err1:
1044	if (udata) {
1045	qedr_free_pbl(dev, pbl_info: &cq->q.pbl_info, pbl: cq->q.pbl_tbl);
1046	ib_umem_release(umem: cq->q.umem);
1047	if (cq->q.db_mmap_entry)
1048	rdma_user_mmap_entry_remove(entry: cq->q.db_mmap_entry);
1049	} else {
1050	dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1051	}
1052	err0:
1053	return -EINVAL;
1054	}
1055
1056	#define QEDR_DESTROY_CQ_MAX_ITERATIONS (10)
1057	#define QEDR_DESTROY_CQ_ITER_DURATION (10)
1058
1059	int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1060	{
1061	struct qedr_dev *dev = get_qedr_dev(ibdev: ibcq->device);
1062	struct qed_rdma_destroy_cq_out_params oparams;
1063	struct qed_rdma_destroy_cq_in_params iparams;
1064	struct qedr_cq *cq = get_qedr_cq(ibcq);
1065	int iter;
1066
1067	DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
1068
1069	cq->destroyed = 1;
1070
1071	/* GSIs CQs are handled by driver, so they don't exist in the FW */
1072	if (cq->cq_type == QEDR_CQ_TYPE_GSI) {
1073	qedr_db_recovery_del(dev, db_addr: cq->db_addr, db_data: &cq->db.data);
1074	return 0;
1075	}
1076
1077	iparams.icid = cq->icid;
1078	dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
1079	dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1080
1081	if (udata) {
1082	qedr_free_pbl(dev, pbl_info: &cq->q.pbl_info, pbl: cq->q.pbl_tbl);
1083	ib_umem_release(umem: cq->q.umem);
1084
1085	if (cq->q.db_rec_data) {
1086	qedr_db_recovery_del(dev, db_addr: cq->q.db_addr,
1087	db_data: &cq->q.db_rec_data->db_data);
1088	rdma_user_mmap_entry_remove(entry: cq->q.db_mmap_entry);
1089	}
1090	} else {
1091	qedr_db_recovery_del(dev, db_addr: cq->db_addr, db_data: &cq->db.data);
1092	}
1093
1094	/* We don't want the IRQ handler to handle a non-existing CQ so we
1095	* wait until all CNQ interrupts, if any, are received. This will always
1096	* happen and will always happen very fast. If not, then a serious error
1097	* has occured. That is why we can use a long delay.
1098	* We spin for a short time so we don’t lose time on context switching
1099	* in case all the completions are handled in that span. Otherwise
1100	* we sleep for a while and check again. Since the CNQ may be
1101	* associated with (only) the current CPU we use msleep to allow the
1102	* current CPU to be freed.
1103	* The CNQ notification is increased in qedr_irq_handler().
1104	*/
1105	iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1106	while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1107	udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1108	iter--;
1109	}
1110
1111	iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1112	while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1113	msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1114	iter--;
1115	}
1116
1117	/* Note that we don't need to have explicit code to wait for the
1118	* completion of the event handler because it is invoked from the EQ.
1119	* Since the destroy CQ ramrod has also been received on the EQ we can
1120	* be certain that there's no event handler in process.
1121	*/
1122	return 0;
1123	}
1124
1125	static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1126	struct ib_qp_attr *attr,
1127	int attr_mask,
1128	struct qed_rdma_modify_qp_in_params
1129	*qp_params)
1130	{
1131	const struct ib_gid_attr *gid_attr;
1132	enum rdma_network_type nw_type;
1133	const struct ib_global_route *grh = rdma_ah_read_grh(attr: &attr->ah_attr);
1134	u32 ipv4_addr;
1135	int ret;
1136	int i;
1137
1138	gid_attr = grh->sgid_attr;
1139	ret = rdma_read_gid_l2_fields(attr: gid_attr, vlan_id: &qp_params->vlan_id, NULL);
1140	if (ret)
1141	return ret;
1142
1143	nw_type = rdma_gid_attr_network_type(attr: gid_attr);
1144	switch (nw_type) {
1145	case RDMA_NETWORK_IPV6:
1146	memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1147	sizeof(qp_params->sgid));
1148	memcpy(&qp_params->dgid.bytes[0],
1149	&grh->dgid,
1150	sizeof(qp_params->dgid));
1151	qp_params->roce_mode = ROCE_V2_IPV6;
1152	SET_FIELD(qp_params->modify_flags,
1153	QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1154	break;
1155	case RDMA_NETWORK_ROCE_V1:
1156	memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1157	sizeof(qp_params->sgid));
1158	memcpy(&qp_params->dgid.bytes[0],
1159	&grh->dgid,
1160	sizeof(qp_params->dgid));
1161	qp_params->roce_mode = ROCE_V1;
1162	break;
1163	case RDMA_NETWORK_IPV4:
1164	memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1165	memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1166	ipv4_addr = qedr_get_ipv4_from_gid(gid: gid_attr->gid.raw);
1167	qp_params->sgid.ipv4_addr = ipv4_addr;
1168	ipv4_addr =
1169	qedr_get_ipv4_from_gid(gid: grh->dgid.raw);
1170	qp_params->dgid.ipv4_addr = ipv4_addr;
1171	SET_FIELD(qp_params->modify_flags,
1172	QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1173	qp_params->roce_mode = ROCE_V2_IPV4;
1174	break;
1175	default:
1176	return -EINVAL;
1177	}
1178
1179	for (i = 0; i < 4; i++) {
1180	qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1181	qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1182	}
1183
1184	if (qp_params->vlan_id >= VLAN_CFI_MASK)
1185	qp_params->vlan_id = 0;
1186
1187	return 0;
1188	}
1189
1190	static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1191	struct ib_qp_init_attr *attrs,
1192	struct ib_udata *udata)
1193	{
1194	struct qedr_device_attr *qattr = &dev->attr;
1195
1196	/* QP0... attrs->qp_type == IB_QPT_GSI */
1197	if (attrs->qp_type != IB_QPT_RC &&
1198	attrs->qp_type != IB_QPT_GSI &&
1199	attrs->qp_type != IB_QPT_XRC_INI &&
1200	attrs->qp_type != IB_QPT_XRC_TGT) {
1201	DP_DEBUG(dev, QEDR_MSG_QP,
1202	"create qp: unsupported qp type=0x%x requested\n",
1203	attrs->qp_type);
1204	return -EOPNOTSUPP;
1205	}
1206
1207	if (attrs->cap.max_send_wr > qattr->max_sqe) {
1208	DP_ERR(dev,
1209	"create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1210	attrs->cap.max_send_wr, qattr->max_sqe);
1211	return -EINVAL;
1212	}
1213
1214	if (attrs->cap.max_inline_data > qattr->max_inline) {
1215	DP_ERR(dev,
1216	"create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1217	attrs->cap.max_inline_data, qattr->max_inline);
1218	return -EINVAL;
1219	}
1220
1221	if (attrs->cap.max_send_sge > qattr->max_sge) {
1222	DP_ERR(dev,
1223	"create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1224	attrs->cap.max_send_sge, qattr->max_sge);
1225	return -EINVAL;
1226	}
1227
1228	if (attrs->cap.max_recv_sge > qattr->max_sge) {
1229	DP_ERR(dev,
1230	"create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1231	attrs->cap.max_recv_sge, qattr->max_sge);
1232	return -EINVAL;
1233	}
1234
1235	/* verify consumer QPs are not trying to use GSI QP's CQ.
1236	* TGT QP isn't associated with RQ/SQ
1237	*/
1238	if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created) &&
1239	(attrs->qp_type != IB_QPT_XRC_TGT) &&
1240	(attrs->qp_type != IB_QPT_XRC_INI)) {
1241	struct qedr_cq *send_cq = get_qedr_cq(ibcq: attrs->send_cq);
1242	struct qedr_cq *recv_cq = get_qedr_cq(ibcq: attrs->recv_cq);
1243
1244	if ((send_cq->cq_type == QEDR_CQ_TYPE_GSI) ||
1245	(recv_cq->cq_type == QEDR_CQ_TYPE_GSI)) {
1246	DP_ERR(dev,
1247	"create qp: consumer QP cannot use GSI CQs.\n");
1248	return -EINVAL;
1249	}
1250	}
1251
1252	return 0;
1253	}
1254
1255	static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1256	struct qedr_srq *srq, struct ib_udata *udata)
1257	{
1258	struct qedr_create_srq_uresp uresp = {};
1259	int rc;
1260
1261	uresp.srq_id = srq->srq_id;
1262
1263	rc = ib_copy_to_udata(udata, src: &uresp, len: sizeof(uresp));
1264	if (rc)
1265	DP_ERR(dev, "create srq: problem copying data to user space\n");
1266
1267	return rc;
1268	}
1269
1270	static void qedr_copy_rq_uresp(struct qedr_dev *dev,
1271	struct qedr_create_qp_uresp *uresp,
1272	struct qedr_qp *qp)
1273	{
1274	/* iWARP requires two doorbells per RQ. */
1275	if (rdma_protocol_iwarp(device: &dev->ibdev, port_num: 1)) {
1276	uresp->rq_db_offset =
1277	DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1278	uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1279	} else {
1280	uresp->rq_db_offset =
1281	DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1282	}
1283
1284	uresp->rq_icid = qp->icid;
1285	if (qp->urq.db_mmap_entry)
1286	uresp->rq_db_rec_addr =
1287	rdma_user_mmap_get_offset(entry: qp->urq.db_mmap_entry);
1288	}
1289
1290	static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1291	struct qedr_create_qp_uresp *uresp,
1292	struct qedr_qp *qp)
1293	{
1294	uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1295
1296	/* iWARP uses the same cid for rq and sq */
1297	if (rdma_protocol_iwarp(device: &dev->ibdev, port_num: 1))
1298	uresp->sq_icid = qp->icid;
1299	else
1300	uresp->sq_icid = qp->icid + 1;
1301
1302	if (qp->usq.db_mmap_entry)
1303	uresp->sq_db_rec_addr =
1304	rdma_user_mmap_get_offset(entry: qp->usq.db_mmap_entry);
1305	}
1306
1307	static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1308	struct qedr_qp *qp, struct ib_udata *udata,
1309	struct qedr_create_qp_uresp *uresp)
1310	{
1311	int rc;
1312
1313	memset(uresp, 0, sizeof(*uresp));
1314
1315	if (qedr_qp_has_sq(qp))
1316	qedr_copy_sq_uresp(dev, uresp, qp);
1317
1318	if (qedr_qp_has_rq(qp))
1319	qedr_copy_rq_uresp(dev, uresp, qp);
1320
1321	uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1322	uresp->qp_id = qp->qp_id;
1323
1324	rc = qedr_ib_copy_to_udata(udata, src: uresp, len: sizeof(*uresp));
1325	if (rc)
1326	DP_ERR(dev,
1327	"create qp: failed a copy to user space with qp icid=0x%x.\n",
1328	qp->icid);
1329
1330	return rc;
1331	}
1332
1333	static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
1334	{
1335	qed_chain_reset(p_chain: &qph->pbl);
1336	qph->prod = 0;
1337	qph->cons = 0;
1338	qph->wqe_cons = 0;
1339	qph->db_data.data.value = cpu_to_le16(0);
1340	}
1341
1342	static void qedr_set_common_qp_params(struct qedr_dev *dev,
1343	struct qedr_qp *qp,
1344	struct qedr_pd *pd,
1345	struct ib_qp_init_attr *attrs)
1346	{
1347	spin_lock_init(&qp->q_lock);
1348	if (rdma_protocol_iwarp(device: &dev->ibdev, port_num: 1)) {
1349	kref_init(kref: &qp->refcnt);
1350	init_completion(x: &qp->iwarp_cm_comp);
1351	init_completion(x: &qp->qp_rel_comp);
1352	}
1353
1354	qp->pd = pd;
1355	qp->qp_type = attrs->qp_type;
1356	qp->max_inline_data = attrs->cap.max_inline_data;
1357	qp->state = QED_ROCE_QP_STATE_RESET;
1358
1359	qp->prev_wqe_size = 0;
1360
1361	qp->signaled = attrs->sq_sig_type == IB_SIGNAL_ALL_WR;
1362	qp->dev = dev;
1363	if (qedr_qp_has_sq(qp)) {
1364	qedr_reset_qp_hwq_info(qph: &qp->sq);
1365	qp->sq.max_sges = attrs->cap.max_send_sge;
1366	qp->sq_cq = get_qedr_cq(ibcq: attrs->send_cq);
1367	DP_DEBUG(dev, QEDR_MSG_QP,
1368	"SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1369	qp->sq.max_sges, qp->sq_cq->icid);
1370	}
1371
1372	if (attrs->srq)
1373	qp->srq = get_qedr_srq(ibsrq: attrs->srq);
1374
1375	if (qedr_qp_has_rq(qp)) {
1376	qedr_reset_qp_hwq_info(qph: &qp->rq);
1377	qp->rq_cq = get_qedr_cq(ibcq: attrs->recv_cq);
1378	qp->rq.max_sges = attrs->cap.max_recv_sge;
1379	DP_DEBUG(dev, QEDR_MSG_QP,
1380	"RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1381	qp->rq.max_sges, qp->rq_cq->icid);
1382	}
1383
1384	DP_DEBUG(dev, QEDR_MSG_QP,
1385	"QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1386	pd->pd_id, qp->qp_type, qp->max_inline_data,
1387	qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1388	DP_DEBUG(dev, QEDR_MSG_QP,
1389	"SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1390	qp->sq.max_sges, qp->sq_cq->icid);
1391	}
1392
1393	static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1394	{
1395	int rc = 0;
1396
1397	if (qedr_qp_has_sq(qp)) {
1398	qp->sq.db = dev->db_addr +
1399	DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1400	qp->sq.db_data.data.icid = qp->icid + 1;
1401	rc = qedr_db_recovery_add(dev, db_addr: qp->sq.db, db_data: &qp->sq.db_data,
1402	db_width: DB_REC_WIDTH_32B, db_space: DB_REC_KERNEL);
1403	if (rc)
1404	return rc;
1405	}
1406
1407	if (qedr_qp_has_rq(qp)) {
1408	qp->rq.db = dev->db_addr +
1409	DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1410	qp->rq.db_data.data.icid = qp->icid;
1411	rc = qedr_db_recovery_add(dev, db_addr: qp->rq.db, db_data: &qp->rq.db_data,
1412	db_width: DB_REC_WIDTH_32B, db_space: DB_REC_KERNEL);
1413	if (rc && qedr_qp_has_sq(qp))
1414	qedr_db_recovery_del(dev, db_addr: qp->sq.db, db_data: &qp->sq.db_data);
1415	}
1416
1417	return rc;
1418	}
1419
1420	static int qedr_check_srq_params(struct qedr_dev *dev,
1421	struct ib_srq_init_attr *attrs,
1422	struct ib_udata *udata)
1423	{
1424	struct qedr_device_attr *qattr = &dev->attr;
1425
1426	if (attrs->attr.max_wr > qattr->max_srq_wr) {
1427	DP_ERR(dev,
1428	"create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1429	attrs->attr.max_wr, qattr->max_srq_wr);
1430	return -EINVAL;
1431	}
1432
1433	if (attrs->attr.max_sge > qattr->max_sge) {
1434	DP_ERR(dev,
1435	"create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1436	attrs->attr.max_sge, qattr->max_sge);
1437	}
1438
1439	if (!udata && attrs->srq_type == IB_SRQT_XRC) {
1440	DP_ERR(dev, "XRC SRQs are not supported in kernel-space\n");
1441	return -EINVAL;
1442	}
1443
1444	return 0;
1445	}
1446
1447	static void qedr_free_srq_user_params(struct qedr_srq *srq)
1448	{
1449	qedr_free_pbl(dev: srq->dev, pbl_info: &srq->usrq.pbl_info, pbl: srq->usrq.pbl_tbl);
1450	ib_umem_release(umem: srq->usrq.umem);
1451	ib_umem_release(umem: srq->prod_umem);
1452	}
1453
1454	static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1455	{
1456	struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1457	struct qedr_dev *dev = srq->dev;
1458
1459	dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1460
1461	dma_free_coherent(dev: &dev->pdev->dev, size: sizeof(struct rdma_srq_producers),
1462	cpu_addr: hw_srq->virt_prod_pair_addr,
1463	dma_handle: hw_srq->phy_prod_pair_addr);
1464	}
1465
1466	static int qedr_init_srq_user_params(struct ib_udata *udata,
1467	struct qedr_srq *srq,
1468	struct qedr_create_srq_ureq *ureq,
1469	int access)
1470	{
1471	struct scatterlist *sg;
1472	int rc;
1473
1474	rc = qedr_init_user_queue(udata, dev: srq->dev, q: &srq->usrq, buf_addr: ureq->srq_addr,
1475	buf_len: ureq->srq_len, requires_db_rec: false, access, alloc_and_init: 1);
1476	if (rc)
1477	return rc;
1478
1479	srq->prod_umem = ib_umem_get(device: srq->ibsrq.device, addr: ureq->prod_pair_addr,
1480	size: sizeof(struct rdma_srq_producers), access);
1481	if (IS_ERR(ptr: srq->prod_umem)) {
1482	qedr_free_pbl(dev: srq->dev, pbl_info: &srq->usrq.pbl_info, pbl: srq->usrq.pbl_tbl);
1483	ib_umem_release(umem: srq->usrq.umem);
1484	DP_ERR(srq->dev,
1485	"create srq: failed ib_umem_get for producer, got %ld\n",
1486	PTR_ERR(srq->prod_umem));
1487	return PTR_ERR(ptr: srq->prod_umem);
1488	}
1489
1490	sg = srq->prod_umem->sgt_append.sgt.sgl;
1491	srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1492
1493	return 0;
1494	}
1495
1496	static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1497	struct qedr_dev *dev,
1498	struct ib_srq_init_attr *init_attr)
1499	{
1500	struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1501	struct qed_chain_init_params params = {
1502	.mode = QED_CHAIN_MODE_PBL,
1503	.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1504	.cnt_type = QED_CHAIN_CNT_TYPE_U32,
1505	.elem_size = QEDR_SRQ_WQE_ELEM_SIZE,
1506	};
1507	dma_addr_t phy_prod_pair_addr;
1508	u32 num_elems;
1509	void *va;
1510	int rc;
1511
1512	va = dma_alloc_coherent(dev: &dev->pdev->dev,
1513	size: sizeof(struct rdma_srq_producers),
1514	dma_handle: &phy_prod_pair_addr, GFP_KERNEL);
1515	if (!va) {
1516	DP_ERR(dev,
1517	"create srq: failed to allocate dma memory for producer\n");
1518	return -ENOMEM;
1519	}
1520
1521	hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1522	hw_srq->virt_prod_pair_addr = va;
1523
1524	num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
1525	params.num_elems = num_elems;
1526
1527	rc = dev->ops->common->chain_alloc(dev->cdev, &hw_srq->pbl, &params);
1528	if (rc)
1529	goto err0;
1530
1531	hw_srq->num_elems = num_elems;
1532
1533	return 0;
1534
1535	err0:
1536	dma_free_coherent(dev: &dev->pdev->dev, size: sizeof(struct rdma_srq_producers),
1537	cpu_addr: va, dma_handle: phy_prod_pair_addr);
1538	return rc;
1539	}
1540
1541	int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1542	struct ib_udata *udata)
1543	{
1544	struct qed_rdma_destroy_srq_in_params destroy_in_params;
1545	struct qed_rdma_create_srq_in_params in_params = {};
1546	struct qedr_dev *dev = get_qedr_dev(ibdev: ibsrq->device);
1547	struct qed_rdma_create_srq_out_params out_params;
1548	struct qedr_pd *pd = get_qedr_pd(ibpd: ibsrq->pd);
1549	struct qedr_create_srq_ureq ureq = {};
1550	u64 pbl_base_addr, phy_prod_pair_addr;
1551	struct qedr_srq_hwq_info *hw_srq;
1552	u32 page_cnt, page_size;
1553	struct qedr_srq *srq = get_qedr_srq(ibsrq);
1554	int rc = 0;
1555
1556	DP_DEBUG(dev, QEDR_MSG_QP,
1557	"create SRQ called from %s (pd %p)\n",
1558	(udata) ? "User lib" : "kernel", pd);
1559
1560	if (init_attr->srq_type != IB_SRQT_BASIC &&
1561	init_attr->srq_type != IB_SRQT_XRC)
1562	return -EOPNOTSUPP;
1563
1564	rc = qedr_check_srq_params(dev, attrs: init_attr, udata);
1565	if (rc)
1566	return -EINVAL;
1567
1568	srq->dev = dev;
1569	srq->is_xrc = (init_attr->srq_type == IB_SRQT_XRC);
1570	hw_srq = &srq->hw_srq;
1571	spin_lock_init(&srq->lock);
1572
1573	hw_srq->max_wr = init_attr->attr.max_wr;
1574	hw_srq->max_sges = init_attr->attr.max_sge;
1575
1576	if (udata) {
1577	if (ib_copy_from_udata(dest: &ureq, udata, min(sizeof(ureq),
1578	udata->inlen))) {
1579	DP_ERR(dev,
1580	"create srq: problem copying data from user space\n");
1581	goto err0;
1582	}
1583
1584	rc = qedr_init_srq_user_params(udata, srq, ureq: &ureq, access: 0);
1585	if (rc)
1586	goto err0;
1587
1588	page_cnt = srq->usrq.pbl_info.num_pbes;
1589	pbl_base_addr = srq->usrq.pbl_tbl->pa;
1590	phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1591	page_size = PAGE_SIZE;
1592	} else {
1593	struct qed_chain *pbl;
1594
1595	rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1596	if (rc)
1597	goto err0;
1598
1599	pbl = &hw_srq->pbl;
1600	page_cnt = qed_chain_get_page_cnt(chain: pbl);
1601	pbl_base_addr = qed_chain_get_pbl_phys(chain: pbl);
1602	phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1603	page_size = QED_CHAIN_PAGE_SIZE;
1604	}
1605
1606	in_params.pd_id = pd->pd_id;
1607	in_params.pbl_base_addr = pbl_base_addr;
1608	in_params.prod_pair_addr = phy_prod_pair_addr;
1609	in_params.num_pages = page_cnt;
1610	in_params.page_size = page_size;
1611	if (srq->is_xrc) {
1612	struct qedr_xrcd *xrcd = get_qedr_xrcd(ibxrcd: init_attr->ext.xrc.xrcd);
1613	struct qedr_cq *cq = get_qedr_cq(ibcq: init_attr->ext.cq);
1614
1615	in_params.is_xrc = 1;
1616	in_params.xrcd_id = xrcd->xrcd_id;
1617	in_params.cq_cid = cq->icid;
1618	}
1619
1620	rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1621	if (rc)
1622	goto err1;
1623
1624	srq->srq_id = out_params.srq_id;
1625
1626	if (udata) {
1627	rc = qedr_copy_srq_uresp(dev, srq, udata);
1628	if (rc)
1629	goto err2;
1630	}
1631
1632	rc = xa_insert_irq(xa: &dev->srqs, index: srq->srq_id, entry: srq, GFP_KERNEL);
1633	if (rc)
1634	goto err2;
1635
1636	DP_DEBUG(dev, QEDR_MSG_SRQ,
1637	"create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
1638	return 0;
1639
1640	err2:
1641	destroy_in_params.srq_id = srq->srq_id;
1642
1643	dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1644	err1:
1645	if (udata)
1646	qedr_free_srq_user_params(srq);
1647	else
1648	qedr_free_srq_kernel_params(srq);
1649	err0:
1650	return -EFAULT;
1651	}
1652
1653	int qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
1654	{
1655	struct qed_rdma_destroy_srq_in_params in_params = {};
1656	struct qedr_dev *dev = get_qedr_dev(ibdev: ibsrq->device);
1657	struct qedr_srq *srq = get_qedr_srq(ibsrq);
1658
1659	xa_erase_irq(xa: &dev->srqs, index: srq->srq_id);
1660	in_params.srq_id = srq->srq_id;
1661	in_params.is_xrc = srq->is_xrc;
1662	dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1663
1664	if (ibsrq->uobject)
1665	qedr_free_srq_user_params(srq);
1666	else
1667	qedr_free_srq_kernel_params(srq);
1668
1669	DP_DEBUG(dev, QEDR_MSG_SRQ,
1670	"destroy srq: destroyed srq with srq_id=0x%0x\n",
1671	srq->srq_id);
1672	return 0;
1673	}
1674
1675	int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1676	enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1677	{
1678	struct qed_rdma_modify_srq_in_params in_params = {};
1679	struct qedr_dev *dev = get_qedr_dev(ibdev: ibsrq->device);
1680	struct qedr_srq *srq = get_qedr_srq(ibsrq);
1681	int rc;
1682
1683	if (attr_mask & IB_SRQ_MAX_WR) {
1684	DP_ERR(dev,
1685	"modify srq: invalid attribute mask=0x%x specified for %p\n",
1686	attr_mask, srq);
1687	return -EINVAL;
1688	}
1689
1690	if (attr_mask & IB_SRQ_LIMIT) {
1691	if (attr->srq_limit >= srq->hw_srq.max_wr) {
1692	DP_ERR(dev,
1693	"modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1694	attr->srq_limit, srq->hw_srq.max_wr);
1695	return -EINVAL;
1696	}
1697
1698	in_params.srq_id = srq->srq_id;
1699	in_params.wqe_limit = attr->srq_limit;
1700	rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1701	if (rc)
1702	return rc;
1703	}
1704
1705	srq->srq_limit = attr->srq_limit;
1706
1707	DP_DEBUG(dev, QEDR_MSG_SRQ,
1708	"modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1709
1710	return 0;
1711	}
1712
1713	static enum qed_rdma_qp_type qedr_ib_to_qed_qp_type(enum ib_qp_type ib_qp_type)
1714	{
1715	switch (ib_qp_type) {
1716	case IB_QPT_RC:
1717	return QED_RDMA_QP_TYPE_RC;
1718	case IB_QPT_XRC_INI:
1719	return QED_RDMA_QP_TYPE_XRC_INI;
1720	case IB_QPT_XRC_TGT:
1721	return QED_RDMA_QP_TYPE_XRC_TGT;
1722	default:
1723	return QED_RDMA_QP_TYPE_INVAL;
1724	}
1725	}
1726
1727	static inline void
1728	qedr_init_common_qp_in_params(struct qedr_dev *dev,
1729	struct qedr_pd *pd,
1730	struct qedr_qp *qp,
1731	struct ib_qp_init_attr *attrs,
1732	bool fmr_and_reserved_lkey,
1733	struct qed_rdma_create_qp_in_params *params)
1734	{
1735	/* QP handle to be written in an async event */
1736	params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1737	params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1738
1739	params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1740	params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1741	params->qp_type = qedr_ib_to_qed_qp_type(ib_qp_type: attrs->qp_type);
1742	params->stats_queue = 0;
1743
1744	if (pd) {
1745	params->pd = pd->pd_id;
1746	params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1747	}
1748
1749	if (qedr_qp_has_sq(qp))
1750	params->sq_cq_id = get_qedr_cq(ibcq: attrs->send_cq)->icid;
1751
1752	if (qedr_qp_has_rq(qp))
1753	params->rq_cq_id = get_qedr_cq(ibcq: attrs->recv_cq)->icid;
1754
1755	if (qedr_qp_has_srq(qp)) {
1756	params->rq_cq_id = get_qedr_cq(ibcq: attrs->recv_cq)->icid;
1757	params->srq_id = qp->srq->srq_id;
1758	params->use_srq = true;
1759	} else {
1760	params->srq_id = 0;
1761	params->use_srq = false;
1762	}
1763	}
1764
1765	static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1766	{
1767	DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1768	"qp=%p. "
1769	"sq_addr=0x%llx, "
1770	"sq_len=%zd, "
1771	"rq_addr=0x%llx, "
1772	"rq_len=%zd"
1773	"\n",
1774	qp,
1775	qedr_qp_has_sq(qp) ? qp->usq.buf_addr : 0x0,
1776	qedr_qp_has_sq(qp) ? qp->usq.buf_len : 0,
1777	qedr_qp_has_rq(qp) ? qp->urq.buf_addr : 0x0,
1778	qedr_qp_has_sq(qp) ? qp->urq.buf_len : 0);
1779	}
1780
1781	static inline void
1782	qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1783	struct qedr_qp *qp,
1784	struct qed_rdma_create_qp_out_params *out_params)
1785	{
1786	qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1787	qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1788
1789	qedr_populate_pbls(dev, umem: qp->usq.umem, pbl: qp->usq.pbl_tbl,
1790	pbl_info: &qp->usq.pbl_info, FW_PAGE_SHIFT);
1791	if (!qp->srq) {
1792	qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1793	qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1794	}
1795
1796	qedr_populate_pbls(dev, umem: qp->urq.umem, pbl: qp->urq.pbl_tbl,
1797	pbl_info: &qp->urq.pbl_info, FW_PAGE_SHIFT);
1798	}
1799
1800	static void qedr_cleanup_user(struct qedr_dev *dev,
1801	struct qedr_ucontext *ctx,
1802	struct qedr_qp *qp)
1803	{
1804	if (qedr_qp_has_sq(qp)) {
1805	ib_umem_release(umem: qp->usq.umem);
1806	qp->usq.umem = NULL;
1807	}
1808
1809	if (qedr_qp_has_rq(qp)) {
1810	ib_umem_release(umem: qp->urq.umem);
1811	qp->urq.umem = NULL;
1812	}
1813
1814	if (rdma_protocol_roce(device: &dev->ibdev, port_num: 1)) {
1815	qedr_free_pbl(dev, pbl_info: &qp->usq.pbl_info, pbl: qp->usq.pbl_tbl);
1816	qedr_free_pbl(dev, pbl_info: &qp->urq.pbl_info, pbl: qp->urq.pbl_tbl);
1817	} else {
1818	kfree(objp: qp->usq.pbl_tbl);
1819	kfree(objp: qp->urq.pbl_tbl);
1820	}
1821
1822	if (qp->usq.db_rec_data) {
1823	qedr_db_recovery_del(dev, db_addr: qp->usq.db_addr,
1824	db_data: &qp->usq.db_rec_data->db_data);
1825	rdma_user_mmap_entry_remove(entry: qp->usq.db_mmap_entry);
1826	}
1827
1828	if (qp->urq.db_rec_data) {
1829	qedr_db_recovery_del(dev, db_addr: qp->urq.db_addr,
1830	db_data: &qp->urq.db_rec_data->db_data);
1831	rdma_user_mmap_entry_remove(entry: qp->urq.db_mmap_entry);
1832	}
1833
1834	if (rdma_protocol_iwarp(device: &dev->ibdev, port_num: 1))
1835	qedr_db_recovery_del(dev, db_addr: qp->urq.db_rec_db2_addr,
1836	db_data: &qp->urq.db_rec_db2_data);
1837	}
1838
1839	static int qedr_create_user_qp(struct qedr_dev *dev,
1840	struct qedr_qp *qp,
1841	struct ib_pd *ibpd,
1842	struct ib_udata *udata,
1843	struct ib_qp_init_attr *attrs)
1844	{
1845	struct qed_rdma_create_qp_in_params in_params;
1846	struct qed_rdma_create_qp_out_params out_params;
1847	struct qedr_create_qp_uresp uresp = {};
1848	struct qedr_create_qp_ureq ureq = {};
1849	int alloc_and_init = rdma_protocol_roce(device: &dev->ibdev, port_num: 1);
1850	struct qedr_ucontext *ctx = NULL;
1851	struct qedr_pd *pd = NULL;
1852	int rc = 0;
1853
1854	qp->create_type = QEDR_QP_CREATE_USER;
1855
1856	if (ibpd) {
1857	pd = get_qedr_pd(ibpd);
1858	ctx = pd->uctx;
1859	}
1860
1861	if (udata) {
1862	rc = ib_copy_from_udata(dest: &ureq, udata, min(sizeof(ureq),
1863	udata->inlen));
1864	if (rc) {
1865	DP_ERR(dev, "Problem copying data from user space\n");
1866	return rc;
1867	}
1868	}
1869
1870	if (qedr_qp_has_sq(qp)) {
1871	/* SQ - read access only (0) */
1872	rc = qedr_init_user_queue(udata, dev, q: &qp->usq, buf_addr: ureq.sq_addr,
1873	buf_len: ureq.sq_len, requires_db_rec: true, access: 0, alloc_and_init);
1874	if (rc)
1875	return rc;
1876	}
1877
1878	if (qedr_qp_has_rq(qp)) {
1879	/* RQ - read access only (0) */
1880	rc = qedr_init_user_queue(udata, dev, q: &qp->urq, buf_addr: ureq.rq_addr,
1881	buf_len: ureq.rq_len, requires_db_rec: true, access: 0, alloc_and_init);
1882	if (rc) {
1883	ib_umem_release(umem: qp->usq.umem);
1884	qp->usq.umem = NULL;
1885	if (rdma_protocol_roce(device: &dev->ibdev, port_num: 1)) {
1886	qedr_free_pbl(dev, pbl_info: &qp->usq.pbl_info,
1887	pbl: qp->usq.pbl_tbl);
1888	} else {
1889	kfree(objp: qp->usq.pbl_tbl);
1890	}
1891	return rc;
1892	}
1893	}
1894
1895	memset(&in_params, 0, sizeof(in_params));
1896	qedr_init_common_qp_in_params(dev, pd, qp, attrs, fmr_and_reserved_lkey: false, params: &in_params);
1897	in_params.qp_handle_lo = ureq.qp_handle_lo;
1898	in_params.qp_handle_hi = ureq.qp_handle_hi;
1899
1900	if (qp->qp_type == IB_QPT_XRC_TGT) {
1901	struct qedr_xrcd *xrcd = get_qedr_xrcd(ibxrcd: attrs->xrcd);
1902
1903	in_params.xrcd_id = xrcd->xrcd_id;
1904	in_params.qp_handle_lo = qp->qp_id;
1905	in_params.use_srq = 1;
1906	}
1907
1908	if (qedr_qp_has_sq(qp)) {
1909	in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1910	in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1911	}
1912
1913	if (qedr_qp_has_rq(qp)) {
1914	in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1915	in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1916	}
1917
1918	if (ctx)
1919	SET_FIELD(in_params.flags, QED_ROCE_EDPM_MODE, ctx->edpm_mode);
1920
1921	qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1922	&in_params, &out_params);
1923
1924	if (!qp->qed_qp) {
1925	rc = -ENOMEM;
1926	goto err1;
1927	}
1928
1929	if (rdma_protocol_iwarp(device: &dev->ibdev, port_num: 1))
1930	qedr_iwarp_populate_user_qp(dev, qp, out_params: &out_params);
1931
1932	qp->qp_id = out_params.qp_id;
1933	qp->icid = out_params.icid;
1934
1935	if (udata) {
1936	rc = qedr_copy_qp_uresp(dev, qp, udata, uresp: &uresp);
1937	if (rc)
1938	goto err;
1939	}
1940
1941	/* db offset was calculated in copy_qp_uresp, now set in the user q */
1942	if (qedr_qp_has_sq(qp)) {
1943	qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset;
1944	qp->sq.max_wr = attrs->cap.max_send_wr;
1945	rc = qedr_db_recovery_add(dev, db_addr: qp->usq.db_addr,
1946	db_data: &qp->usq.db_rec_data->db_data,
1947	db_width: DB_REC_WIDTH_32B,
1948	db_space: DB_REC_USER);
1949	if (rc)
1950	goto err;
1951	}
1952
1953	if (qedr_qp_has_rq(qp)) {
1954	qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset;
1955	qp->rq.max_wr = attrs->cap.max_recv_wr;
1956	rc = qedr_db_recovery_add(dev, db_addr: qp->urq.db_addr,
1957	db_data: &qp->urq.db_rec_data->db_data,
1958	db_width: DB_REC_WIDTH_32B,
1959	db_space: DB_REC_USER);
1960	if (rc)
1961	goto err;
1962	}
1963
1964	if (rdma_protocol_iwarp(device: &dev->ibdev, port_num: 1)) {
1965	qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset;
1966
1967	/* calculate the db_rec_db2 data since it is constant so no
1968	* need to reflect from user
1969	*/
1970	qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid);
1971	qp->urq.db_rec_db2_data.data.value =
1972	cpu_to_le16(DQ_TCM_IWARP_POST_RQ_CF_CMD);
1973
1974	rc = qedr_db_recovery_add(dev, db_addr: qp->urq.db_rec_db2_addr,
1975	db_data: &qp->urq.db_rec_db2_data,
1976	db_width: DB_REC_WIDTH_32B,
1977	db_space: DB_REC_USER);
1978	if (rc)
1979	goto err;
1980	}
1981	qedr_qp_user_print(dev, qp);
1982	return rc;
1983	err:
1984	rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1985	if (rc)
1986	DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1987
1988	err1:
1989	qedr_cleanup_user(dev, ctx, qp);
1990	return rc;
1991	}
1992
1993	static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1994	{
1995	int rc;
1996
1997	qp->sq.db = dev->db_addr +
1998	DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1999	qp->sq.db_data.data.icid = qp->icid;
2000
2001	rc = qedr_db_recovery_add(dev, db_addr: qp->sq.db,
2002	db_data: &qp->sq.db_data,
2003	db_width: DB_REC_WIDTH_32B,
2004	db_space: DB_REC_KERNEL);
2005	if (rc)
2006	return rc;
2007
2008	qp->rq.db = dev->db_addr +
2009	DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
2010	qp->rq.db_data.data.icid = qp->icid;
2011	qp->rq.iwarp_db2 = dev->db_addr +
2012	DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
2013	qp->rq.iwarp_db2_data.data.icid = qp->icid;
2014	qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
2015
2016	rc = qedr_db_recovery_add(dev, db_addr: qp->rq.db,
2017	db_data: &qp->rq.db_data,
2018	db_width: DB_REC_WIDTH_32B,
2019	db_space: DB_REC_KERNEL);
2020	if (rc)
2021	return rc;
2022
2023	rc = qedr_db_recovery_add(dev, db_addr: qp->rq.iwarp_db2,
2024	db_data: &qp->rq.iwarp_db2_data,
2025	db_width: DB_REC_WIDTH_32B,
2026	db_space: DB_REC_KERNEL);
2027	return rc;
2028	}
2029
2030	static int
2031	qedr_roce_create_kernel_qp(struct qedr_dev *dev,
2032	struct qedr_qp *qp,
2033	struct qed_rdma_create_qp_in_params *in_params,
2034	u32 n_sq_elems, u32 n_rq_elems)
2035	{
2036	struct qed_rdma_create_qp_out_params out_params;
2037	struct qed_chain_init_params params = {
2038	.mode = QED_CHAIN_MODE_PBL,
2039	.cnt_type = QED_CHAIN_CNT_TYPE_U32,
2040	};
2041	int rc;
2042
2043	params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
2044	params.num_elems = n_sq_elems;
2045	params.elem_size = QEDR_SQE_ELEMENT_SIZE;
2046
2047	rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);
2048	if (rc)
2049	return rc;
2050
2051	in_params->sq_num_pages = qed_chain_get_page_cnt(chain: &qp->sq.pbl);
2052	in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(chain: &qp->sq.pbl);
2053
2054	params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
2055	params.num_elems = n_rq_elems;
2056	params.elem_size = QEDR_RQE_ELEMENT_SIZE;
2057
2058	rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);
2059	if (rc)
2060	return rc;
2061
2062	in_params->rq_num_pages = qed_chain_get_page_cnt(chain: &qp->rq.pbl);
2063	in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(chain: &qp->rq.pbl);
2064
2065	qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
2066	in_params, &out_params);
2067
2068	if (!qp->qed_qp)
2069	return -EINVAL;
2070
2071	qp->qp_id = out_params.qp_id;
2072	qp->icid = out_params.icid;
2073
2074	return qedr_set_roce_db_info(dev, qp);
2075	}
2076
2077	static int
2078	qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
2079	struct qedr_qp *qp,
2080	struct qed_rdma_create_qp_in_params *in_params,
2081	u32 n_sq_elems, u32 n_rq_elems)
2082	{
2083	struct qed_rdma_create_qp_out_params out_params;
2084	struct qed_chain_init_params params = {
2085	.mode = QED_CHAIN_MODE_PBL,
2086	.cnt_type = QED_CHAIN_CNT_TYPE_U32,
2087	};
2088	int rc;
2089
2090	in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
2091	QEDR_SQE_ELEMENT_SIZE,
2092	QED_CHAIN_PAGE_SIZE,
2093	QED_CHAIN_MODE_PBL);
2094	in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
2095	QEDR_RQE_ELEMENT_SIZE,
2096	QED_CHAIN_PAGE_SIZE,
2097	QED_CHAIN_MODE_PBL);
2098
2099	qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
2100	in_params, &out_params);
2101
2102	if (!qp->qed_qp)
2103	return -EINVAL;
2104
2105	/* Now we allocate the chain */
2106
2107	params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
2108	params.num_elems = n_sq_elems;
2109	params.elem_size = QEDR_SQE_ELEMENT_SIZE;
2110	params.ext_pbl_virt = out_params.sq_pbl_virt;
2111	params.ext_pbl_phys = out_params.sq_pbl_phys;
2112
2113	rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);
2114	if (rc)
2115	goto err;
2116
2117	params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
2118	params.num_elems = n_rq_elems;
2119	params.elem_size = QEDR_RQE_ELEMENT_SIZE;
2120	params.ext_pbl_virt = out_params.rq_pbl_virt;
2121	params.ext_pbl_phys = out_params.rq_pbl_phys;
2122
2123	rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);
2124	if (rc)
2125	goto err;
2126
2127	qp->qp_id = out_params.qp_id;
2128	qp->icid = out_params.icid;
2129
2130	return qedr_set_iwarp_db_info(dev, qp);
2131
2132	err:
2133	dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2134
2135	return rc;
2136	}
2137
2138	static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
2139	{
2140	dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
2141	kfree(objp: qp->wqe_wr_id);
2142
2143	dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
2144	kfree(objp: qp->rqe_wr_id);
2145
2146	/* GSI qp is not registered to db mechanism so no need to delete */
2147	if (qp->qp_type == IB_QPT_GSI)
2148	return;
2149
2150	qedr_db_recovery_del(dev, db_addr: qp->sq.db, db_data: &qp->sq.db_data);
2151
2152	if (!qp->srq) {
2153	qedr_db_recovery_del(dev, db_addr: qp->rq.db, db_data: &qp->rq.db_data);
2154
2155	if (rdma_protocol_iwarp(device: &dev->ibdev, port_num: 1))
2156	qedr_db_recovery_del(dev, db_addr: qp->rq.iwarp_db2,
2157	db_data: &qp->rq.iwarp_db2_data);
2158	}
2159	}
2160
2161	static int qedr_create_kernel_qp(struct qedr_dev *dev,
2162	struct qedr_qp *qp,
2163	struct ib_pd *ibpd,
2164	struct ib_qp_init_attr *attrs)
2165	{
2166	struct qed_rdma_create_qp_in_params in_params;
2167	struct qedr_pd *pd = get_qedr_pd(ibpd);
2168	int rc = -EINVAL;
2169	u32 n_rq_elems;
2170	u32 n_sq_elems;
2171	u32 n_sq_entries;
2172
2173	memset(&in_params, 0, sizeof(in_params));
2174	qp->create_type = QEDR_QP_CREATE_KERNEL;
2175
2176	/* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
2177	* the ring. The ring should allow at least a single WR, even if the
2178	* user requested none, due to allocation issues.
2179	* We should add an extra WR since the prod and cons indices of
2180	* wqe_wr_id are managed in such a way that the WQ is considered full
2181	* when (prod+1)%max_wr==cons. We currently don't do that because we
2182	* double the number of entries due an iSER issue that pushes far more
2183	* WRs than indicated. If we decline its ib_post_send() then we get
2184	* error prints in the dmesg we'd like to avoid.
2185	*/
2186	qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
2187	dev->attr.max_sqe);
2188
2189	qp->wqe_wr_id = kcalloc(n: qp->sq.max_wr, size: sizeof(*qp->wqe_wr_id),
2190	GFP_KERNEL);
2191	if (!qp->wqe_wr_id) {
2192	DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
2193	return -ENOMEM;
2194	}
2195
2196	/* QP handle to be written in CQE */
2197	in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
2198	in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
2199
2200	/* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
2201	* the ring. There ring should allow at least a single WR, even if the
2202	* user requested none, due to allocation issues.
2203	*/
2204	qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
2205
2206	/* Allocate driver internal RQ array */
2207	qp->rqe_wr_id = kcalloc(n: qp->rq.max_wr, size: sizeof(*qp->rqe_wr_id),
2208	GFP_KERNEL);
2209	if (!qp->rqe_wr_id) {
2210	DP_ERR(dev,
2211	"create qp: failed RQ shadow memory allocation\n");
2212	kfree(objp: qp->wqe_wr_id);
2213	return -ENOMEM;
2214	}
2215
2216	qedr_init_common_qp_in_params(dev, pd, qp, attrs, fmr_and_reserved_lkey: true, params: &in_params);
2217
2218	n_sq_entries = attrs->cap.max_send_wr;
2219	n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
2220	n_sq_entries = max_t(u32, n_sq_entries, 1);
2221	n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
2222
2223	n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
2224
2225	if (rdma_protocol_iwarp(device: &dev->ibdev, port_num: 1))
2226	rc = qedr_iwarp_create_kernel_qp(dev, qp, in_params: &in_params,
2227	n_sq_elems, n_rq_elems);
2228	else
2229	rc = qedr_roce_create_kernel_qp(dev, qp, in_params: &in_params,
2230	n_sq_elems, n_rq_elems);
2231	if (rc)
2232	qedr_cleanup_kernel(dev, qp);
2233
2234	return rc;
2235	}
2236
2237	static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp,
2238	struct ib_udata *udata)
2239	{
2240	struct qedr_ucontext *ctx =
2241	rdma_udata_to_drv_context(udata, struct qedr_ucontext,
2242	ibucontext);
2243	int rc;
2244
2245	if (qp->qp_type != IB_QPT_GSI) {
2246	rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2247	if (rc)
2248	return rc;
2249	}
2250
2251	if (qp->create_type == QEDR_QP_CREATE_USER)
2252	qedr_cleanup_user(dev, ctx, qp);
2253	else
2254	qedr_cleanup_kernel(dev, qp);
2255
2256	return 0;
2257	}
2258
2259	int qedr_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
2260	struct ib_udata *udata)
2261	{
2262	struct qedr_xrcd *xrcd = NULL;
2263	struct ib_pd *ibpd = ibqp->pd;
2264	struct qedr_pd *pd = get_qedr_pd(ibpd);
2265	struct qedr_dev *dev = get_qedr_dev(ibdev: ibqp->device);
2266	struct qedr_qp *qp = get_qedr_qp(ibqp);
2267	int rc = 0;
2268
2269	if (attrs->create_flags)
2270	return -EOPNOTSUPP;
2271
2272	if (attrs->qp_type == IB_QPT_XRC_TGT)
2273	xrcd = get_qedr_xrcd(ibxrcd: attrs->xrcd);
2274	else
2275	pd = get_qedr_pd(ibpd);
2276
2277	DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
2278	udata ? "user library" : "kernel", pd);
2279
2280	rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata);
2281	if (rc)
2282	return rc;
2283
2284	DP_DEBUG(dev, QEDR_MSG_QP,
2285	"create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
2286	udata ? "user library" : "kernel", attrs->event_handler, pd,
2287	get_qedr_cq(attrs->send_cq),
2288	get_qedr_cq(attrs->send_cq)->icid,
2289	get_qedr_cq(attrs->recv_cq),
2290	attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
2291
2292	qedr_set_common_qp_params(dev, qp, pd, attrs);
2293
2294	if (attrs->qp_type == IB_QPT_GSI)
2295	return qedr_create_gsi_qp(dev, attrs, qp);
2296
2297	if (udata || xrcd)
2298	rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
2299	else
2300	rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
2301
2302	if (rc)
2303	return rc;
2304
2305	qp->ibqp.qp_num = qp->qp_id;
2306
2307	if (rdma_protocol_iwarp(device: &dev->ibdev, port_num: 1)) {
2308	rc = xa_insert(xa: &dev->qps, index: qp->qp_id, entry: qp, GFP_KERNEL);
2309	if (rc)
2310	goto out_free_qp_resources;
2311	}
2312
2313	return 0;
2314
2315	out_free_qp_resources:
2316	qedr_free_qp_resources(dev, qp, udata);
2317	return -EFAULT;
2318	}
2319
2320	static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
2321	{
2322	switch (qp_state) {
2323	case QED_ROCE_QP_STATE_RESET:
2324	return IB_QPS_RESET;
2325	case QED_ROCE_QP_STATE_INIT:
2326	return IB_QPS_INIT;
2327	case QED_ROCE_QP_STATE_RTR:
2328	return IB_QPS_RTR;
2329	case QED_ROCE_QP_STATE_RTS:
2330	return IB_QPS_RTS;
2331	case QED_ROCE_QP_STATE_SQD:
2332	return IB_QPS_SQD;
2333	case QED_ROCE_QP_STATE_ERR:
2334	return IB_QPS_ERR;
2335	case QED_ROCE_QP_STATE_SQE:
2336	return IB_QPS_SQE;
2337	}
2338	return IB_QPS_ERR;
2339	}
2340
2341	static enum qed_roce_qp_state qedr_get_state_from_ibqp(
2342	enum ib_qp_state qp_state)
2343	{
2344	switch (qp_state) {
2345	case IB_QPS_RESET:
2346	return QED_ROCE_QP_STATE_RESET;
2347	case IB_QPS_INIT:
2348	return QED_ROCE_QP_STATE_INIT;
2349	case IB_QPS_RTR:
2350	return QED_ROCE_QP_STATE_RTR;
2351	case IB_QPS_RTS:
2352	return QED_ROCE_QP_STATE_RTS;
2353	case IB_QPS_SQD:
2354	return QED_ROCE_QP_STATE_SQD;
2355	case IB_QPS_ERR:
2356	return QED_ROCE_QP_STATE_ERR;
2357	default:
2358	return QED_ROCE_QP_STATE_ERR;
2359	}
2360	}
2361
2362	static int qedr_update_qp_state(struct qedr_dev *dev,
2363	struct qedr_qp *qp,
2364	enum qed_roce_qp_state cur_state,
2365	enum qed_roce_qp_state new_state)
2366	{
2367	int status = 0;
2368
2369	if (new_state == cur_state)
2370	return 0;
2371
2372	switch (cur_state) {
2373	case QED_ROCE_QP_STATE_RESET:
2374	switch (new_state) {
2375	case QED_ROCE_QP_STATE_INIT:
2376	break;
2377	default:
2378	status = -EINVAL;
2379	break;
2380	}
2381	break;
2382	case QED_ROCE_QP_STATE_INIT:
2383	switch (new_state) {
2384	case QED_ROCE_QP_STATE_RTR:
2385	/* Update doorbell (in case post_recv was
2386	* done before move to RTR)
2387	*/
2388
2389	if (rdma_protocol_roce(device: &dev->ibdev, port_num: 1)) {
2390	writel(val: qp->rq.db_data.raw, addr: qp->rq.db);
2391	}
2392	break;
2393	case QED_ROCE_QP_STATE_ERR:
2394	break;
2395	default:
2396	/* Invalid state change. */
2397	status = -EINVAL;
2398	break;
2399	}
2400	break;
2401	case QED_ROCE_QP_STATE_RTR:
2402	/* RTR->XXX */
2403	switch (new_state) {
2404	case QED_ROCE_QP_STATE_RTS:
2405	break;
2406	case QED_ROCE_QP_STATE_ERR:
2407	break;
2408	default:
2409	/* Invalid state change. */
2410	status = -EINVAL;
2411	break;
2412	}
2413	break;
2414	case QED_ROCE_QP_STATE_RTS:
2415	/* RTS->XXX */
2416	switch (new_state) {
2417	case QED_ROCE_QP_STATE_SQD:
2418	break;
2419	case QED_ROCE_QP_STATE_ERR:
2420	break;
2421	default:
2422	/* Invalid state change. */
2423	status = -EINVAL;
2424	break;
2425	}
2426	break;
2427	case QED_ROCE_QP_STATE_SQD:
2428	/* SQD->XXX */
2429	switch (new_state) {
2430	case QED_ROCE_QP_STATE_RTS:
2431	case QED_ROCE_QP_STATE_ERR:
2432	break;
2433	default:
2434	/* Invalid state change. */
2435	status = -EINVAL;
2436	break;
2437	}
2438	break;
2439	case QED_ROCE_QP_STATE_ERR:
2440	/* ERR->XXX */
2441	switch (new_state) {
2442	case QED_ROCE_QP_STATE_RESET:
2443	if ((qp->rq.prod != qp->rq.cons) ||
2444	(qp->sq.prod != qp->sq.cons)) {
2445	DP_NOTICE(dev,
2446	"Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2447	qp->rq.prod, qp->rq.cons, qp->sq.prod,
2448	qp->sq.cons);
2449	status = -EINVAL;
2450	}
2451	break;
2452	default:
2453	status = -EINVAL;
2454	break;
2455	}
2456	break;
2457	default:
2458	status = -EINVAL;
2459	break;
2460	}
2461
2462	return status;
2463	}
2464
2465	int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2466	int attr_mask, struct ib_udata *udata)
2467	{
2468	struct qedr_qp *qp = get_qedr_qp(ibqp);
2469	struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2470	struct qedr_dev *dev = get_qedr_dev(ibdev: &qp->dev->ibdev);
2471	const struct ib_global_route *grh = rdma_ah_read_grh(attr: &attr->ah_attr);
2472	enum ib_qp_state old_qp_state, new_qp_state;
2473	enum qed_roce_qp_state cur_state;
2474	int rc = 0;
2475
2476	DP_DEBUG(dev, QEDR_MSG_QP,
2477	"modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2478	attr->qp_state);
2479
2480	if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
2481	return -EOPNOTSUPP;
2482
2483	old_qp_state = qedr_get_ibqp_state(qp_state: qp->state);
2484	if (attr_mask & IB_QP_STATE)
2485	new_qp_state = attr->qp_state;
2486	else
2487	new_qp_state = old_qp_state;
2488
2489	if (rdma_protocol_roce(device: &dev->ibdev, port_num: 1)) {
2490	if (!ib_modify_qp_is_ok(cur_state: old_qp_state, next_state: new_qp_state,
2491	type: ibqp->qp_type, mask: attr_mask)) {
2492	DP_ERR(dev,
2493	"modify qp: invalid attribute mask=0x%x specified for\n"
2494	"qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2495	attr_mask, qp->qp_id, ibqp->qp_type,
2496	old_qp_state, new_qp_state);
2497	rc = -EINVAL;
2498	goto err;
2499	}
2500	}
2501
2502	/* Translate the masks... */
2503	if (attr_mask & IB_QP_STATE) {
2504	SET_FIELD(qp_params.modify_flags,
2505	QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2506	qp_params.new_state = qedr_get_state_from_ibqp(qp_state: attr->qp_state);
2507	}
2508
2509	if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2510	qp_params.sqd_async = true;
2511
2512	if (attr_mask & IB_QP_PKEY_INDEX) {
2513	SET_FIELD(qp_params.modify_flags,
2514	QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2515	if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2516	rc = -EINVAL;
2517	goto err;
2518	}
2519
2520	qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2521	}
2522
2523	if (attr_mask & IB_QP_QKEY)
2524	qp->qkey = attr->qkey;
2525
2526	if (attr_mask & IB_QP_ACCESS_FLAGS) {
2527	SET_FIELD(qp_params.modify_flags,
2528	QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2529	qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2530	IB_ACCESS_REMOTE_READ;
2531	qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2532	IB_ACCESS_REMOTE_WRITE;
2533	qp_params.incoming_atomic_en = attr->qp_access_flags &
2534	IB_ACCESS_REMOTE_ATOMIC;
2535	}
2536
2537	if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
2538	if (rdma_protocol_iwarp(device: &dev->ibdev, port_num: 1))
2539	return -EINVAL;
2540
2541	if (attr_mask & IB_QP_PATH_MTU) {
2542	if (attr->path_mtu < IB_MTU_256 ||
2543	attr->path_mtu > IB_MTU_4096) {
2544	pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2545	rc = -EINVAL;
2546	goto err;
2547	}
2548	qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2549	ib_mtu_enum_to_int(iboe_get_mtu
2550	(dev->ndev->mtu)));
2551	}
2552
2553	if (!qp->mtu) {
2554	qp->mtu =
2555	ib_mtu_enum_to_int(mtu: iboe_get_mtu(mtu: dev->ndev->mtu));
2556	pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2557	}
2558
2559	SET_FIELD(qp_params.modify_flags,
2560	QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2561
2562	qp_params.traffic_class_tos = grh->traffic_class;
2563	qp_params.flow_label = grh->flow_label;
2564	qp_params.hop_limit_ttl = grh->hop_limit;
2565
2566	qp->sgid_idx = grh->sgid_index;
2567
2568	rc = get_gid_info_from_table(ibqp, attr, attr_mask, qp_params: &qp_params);
2569	if (rc) {
2570	DP_ERR(dev,
2571	"modify qp: problems with GID index %d (rc=%d)\n",
2572	grh->sgid_index, rc);
2573	return rc;
2574	}
2575
2576	rc = qedr_get_dmac(dev, ah_attr: &attr->ah_attr,
2577	mac_addr: qp_params.remote_mac_addr);
2578	if (rc)
2579	return rc;
2580
2581	qp_params.use_local_mac = true;
2582	ether_addr_copy(dst: qp_params.local_mac_addr, src: dev->ndev->dev_addr);
2583
2584	DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2585	qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2586	qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2587	DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2588	qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2589	qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2590	DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2591	qp_params.remote_mac_addr);
2592
2593	qp_params.mtu = qp->mtu;
2594	qp_params.lb_indication = false;
2595	}
2596
2597	if (!qp_params.mtu) {
2598	/* Stay with current MTU */
2599	if (qp->mtu)
2600	qp_params.mtu = qp->mtu;
2601	else
2602	qp_params.mtu =
2603	ib_mtu_enum_to_int(mtu: iboe_get_mtu(mtu: dev->ndev->mtu));
2604	}
2605
2606	if (attr_mask & IB_QP_TIMEOUT) {
2607	SET_FIELD(qp_params.modify_flags,
2608	QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2609
2610	/* The received timeout value is an exponent used like this:
2611	* "12.7.34 LOCAL ACK TIMEOUT
2612	* Value representing the transport (ACK) timeout for use by
2613	* the remote, expressed as: 4.096 * 2^timeout [usec]"
2614	* The FW expects timeout in msec so we need to divide the usec
2615	* result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
2616	* so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
2617	* The value of zero means infinite so we use a 'max_t' to make
2618	* sure that sub 1 msec values will be configured as 1 msec.
2619	*/
2620	if (attr->timeout)
2621	qp_params.ack_timeout =
2622	1 << max_t(int, attr->timeout - 8, 0);
2623	else
2624	qp_params.ack_timeout = 0;
2625
2626	qp->timeout = attr->timeout;
2627	}
2628
2629	if (attr_mask & IB_QP_RETRY_CNT) {
2630	SET_FIELD(qp_params.modify_flags,
2631	QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2632	qp_params.retry_cnt = attr->retry_cnt;
2633	}
2634
2635	if (attr_mask & IB_QP_RNR_RETRY) {
2636	SET_FIELD(qp_params.modify_flags,
2637	QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2638	qp_params.rnr_retry_cnt = attr->rnr_retry;
2639	}
2640
2641	if (attr_mask & IB_QP_RQ_PSN) {
2642	SET_FIELD(qp_params.modify_flags,
2643	QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2644	qp_params.rq_psn = attr->rq_psn;
2645	qp->rq_psn = attr->rq_psn;
2646	}
2647
2648	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2649	if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2650	rc = -EINVAL;
2651	DP_ERR(dev,
2652	"unsupported max_rd_atomic=%d, supported=%d\n",
2653	attr->max_rd_atomic,
2654	dev->attr.max_qp_req_rd_atomic_resc);
2655	goto err;
2656	}
2657
2658	SET_FIELD(qp_params.modify_flags,
2659	QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2660	qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2661	}
2662
2663	if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2664	SET_FIELD(qp_params.modify_flags,
2665	QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2666	qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2667	}
2668
2669	if (attr_mask & IB_QP_SQ_PSN) {
2670	SET_FIELD(qp_params.modify_flags,
2671	QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2672	qp_params.sq_psn = attr->sq_psn;
2673	qp->sq_psn = attr->sq_psn;
2674	}
2675
2676	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2677	if (attr->max_dest_rd_atomic >
2678	dev->attr.max_qp_resp_rd_atomic_resc) {
2679	DP_ERR(dev,
2680	"unsupported max_dest_rd_atomic=%d, supported=%d\n",
2681	attr->max_dest_rd_atomic,
2682	dev->attr.max_qp_resp_rd_atomic_resc);
2683
2684	rc = -EINVAL;
2685	goto err;
2686	}
2687
2688	SET_FIELD(qp_params.modify_flags,
2689	QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2690	qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2691	}
2692
2693	if (attr_mask & IB_QP_DEST_QPN) {
2694	SET_FIELD(qp_params.modify_flags,
2695	QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2696
2697	qp_params.dest_qp = attr->dest_qp_num;
2698	qp->dest_qp_num = attr->dest_qp_num;
2699	}
2700
2701	cur_state = qp->state;
2702
2703	/* Update the QP state before the actual ramrod to prevent a race with
2704	* fast path. Modifying the QP state to error will cause the device to
2705	* flush the CQEs and while polling the flushed CQEs will considered as
2706	* a potential issue if the QP isn't in error state.
2707	*/
2708	if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2709	!udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2710	qp->state = QED_ROCE_QP_STATE_ERR;
2711
2712	if (qp->qp_type != IB_QPT_GSI)
2713	rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2714	qp->qed_qp, &qp_params);
2715
2716	if (attr_mask & IB_QP_STATE) {
2717	if ((qp->qp_type != IB_QPT_GSI) && (!udata))
2718	rc = qedr_update_qp_state(dev, qp, cur_state,
2719	new_state: qp_params.new_state);
2720	qp->state = qp_params.new_state;
2721	}
2722
2723	err:
2724	return rc;
2725	}
2726
2727	static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2728	{
2729	int ib_qp_acc_flags = 0;
2730
2731	if (params->incoming_rdma_write_en)
2732	ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2733	if (params->incoming_rdma_read_en)
2734	ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2735	if (params->incoming_atomic_en)
2736	ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2737	ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2738	return ib_qp_acc_flags;
2739	}
2740
2741	int qedr_query_qp(struct ib_qp *ibqp,
2742	struct ib_qp_attr *qp_attr,
2743	int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2744	{
2745	struct qed_rdma_query_qp_out_params params;
2746	struct qedr_qp *qp = get_qedr_qp(ibqp);
2747	struct qedr_dev *dev = qp->dev;
2748	int rc = 0;
2749
2750	memset(&params, 0, sizeof(params));
2751	memset(qp_attr, 0, sizeof(*qp_attr));
2752	memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2753
2754	if (qp->qp_type != IB_QPT_GSI) {
2755	rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2756	if (rc)
2757	goto err;
2758	qp_attr->qp_state = qedr_get_ibqp_state(qp_state: params.state);
2759	} else {
2760	qp_attr->qp_state = qedr_get_ibqp_state(qp_state: QED_ROCE_QP_STATE_RTS);
2761	}
2762
2763	qp_attr->cur_qp_state = qedr_get_ibqp_state(qp_state: params.state);
2764	qp_attr->path_mtu = ib_mtu_int_to_enum(mtu: params.mtu);
2765	qp_attr->path_mig_state = IB_MIG_MIGRATED;
2766	qp_attr->rq_psn = params.rq_psn;
2767	qp_attr->sq_psn = params.sq_psn;
2768	qp_attr->dest_qp_num = params.dest_qp;
2769
2770	qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(params: &params);
2771
2772	qp_attr->cap.max_send_wr = qp->sq.max_wr;
2773	qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2774	qp_attr->cap.max_send_sge = qp->sq.max_sges;
2775	qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2776	qp_attr->cap.max_inline_data = dev->attr.max_inline;
2777	qp_init_attr->cap = qp_attr->cap;
2778
2779	qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2780	rdma_ah_set_grh(attr: &qp_attr->ah_attr, NULL,
2781	flow_label: params.flow_label, sgid_index: qp->sgid_idx,
2782	hop_limit: params.hop_limit_ttl, traffic_class: params.traffic_class_tos);
2783	rdma_ah_set_dgid_raw(attr: &qp_attr->ah_attr, dgid: &params.dgid.bytes[0]);
2784	rdma_ah_set_port_num(attr: &qp_attr->ah_attr, port_num: 1);
2785	rdma_ah_set_sl(attr: &qp_attr->ah_attr, sl: 0);
2786	qp_attr->timeout = qp->timeout;
2787	qp_attr->rnr_retry = params.rnr_retry;
2788	qp_attr->retry_cnt = params.retry_cnt;
2789	qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2790	qp_attr->pkey_index = params.pkey_index;
2791	qp_attr->port_num = 1;
2792	rdma_ah_set_path_bits(attr: &qp_attr->ah_attr, src_path_bits: 0);
2793	rdma_ah_set_static_rate(attr: &qp_attr->ah_attr, static_rate: 0);
2794	qp_attr->alt_pkey_index = 0;
2795	qp_attr->alt_port_num = 0;
2796	qp_attr->alt_timeout = 0;
2797	memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2798
2799	qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2800	qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2801	qp_attr->max_rd_atomic = params.max_rd_atomic;
2802	qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2803
2804	DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2805	qp_attr->cap.max_inline_data);
2806
2807	err:
2808	return rc;
2809	}
2810
2811	int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
2812	{
2813	struct qedr_qp *qp = get_qedr_qp(ibqp);
2814	struct qedr_dev *dev = qp->dev;
2815	struct ib_qp_attr attr;
2816	int attr_mask = 0;
2817
2818	DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2819	qp, qp->qp_type);
2820
2821	if (rdma_protocol_roce(device: &dev->ibdev, port_num: 1)) {
2822	if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2823	(qp->state != QED_ROCE_QP_STATE_ERR) &&
2824	(qp->state != QED_ROCE_QP_STATE_INIT)) {
2825
2826	attr.qp_state = IB_QPS_ERR;
2827	attr_mask |= IB_QP_STATE;
2828
2829	/* Change the QP state to ERROR */
2830	qedr_modify_qp(ibqp, attr: &attr, attr_mask, NULL);
2831	}
2832	} else {
2833	/* If connection establishment started the WAIT_FOR_CONNECT
2834	* bit will be on and we need to Wait for the establishment
2835	* to complete before destroying the qp.
2836	*/
2837	if (test_and_set_bit(nr: QEDR_IWARP_CM_WAIT_FOR_CONNECT,
2838	addr: &qp->iwarp_cm_flags))
2839	wait_for_completion(&qp->iwarp_cm_comp);
2840
2841	/* If graceful disconnect started, the WAIT_FOR_DISCONNECT
2842	* bit will be on, and we need to wait for the disconnect to
2843	* complete before continuing. We can use the same completion,
2844	* iwarp_cm_comp, since this is the only place that waits for
2845	* this completion and it is sequential. In addition,
2846	* disconnect can't occur before the connection is fully
2847	* established, therefore if WAIT_FOR_DISCONNECT is on it
2848	* means WAIT_FOR_CONNECT is also on and the completion for
2849	* CONNECT already occurred.
2850	*/
2851	if (test_and_set_bit(nr: QEDR_IWARP_CM_WAIT_FOR_DISCONNECT,
2852	addr: &qp->iwarp_cm_flags))
2853	wait_for_completion(&qp->iwarp_cm_comp);
2854	}
2855
2856	if (qp->qp_type == IB_QPT_GSI)
2857	qedr_destroy_gsi_qp(dev);
2858
2859	/* We need to remove the entry from the xarray before we release the
2860	* qp_id to avoid a race of the qp_id being reallocated and failing
2861	* on xa_insert
2862	*/
2863	if (rdma_protocol_iwarp(device: &dev->ibdev, port_num: 1))
2864	xa_erase(&dev->qps, index: qp->qp_id);
2865
2866	qedr_free_qp_resources(dev, qp, udata);
2867
2868	if (rdma_protocol_iwarp(device: &dev->ibdev, port_num: 1)) {
2869	qedr_iw_qp_rem_ref(qp: &qp->ibqp);
2870	wait_for_completion(&qp->qp_rel_comp);
2871	}
2872
2873	return 0;
2874	}
2875
2876	int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
2877	struct ib_udata *udata)
2878	{
2879	struct qedr_ah *ah = get_qedr_ah(ibah);
2880
2881	rdma_copy_ah_attr(dest: &ah->attr, src: init_attr->ah_attr);
2882
2883	return 0;
2884	}
2885
2886	int qedr_destroy_ah(struct ib_ah *ibah, u32 flags)
2887	{
2888	struct qedr_ah *ah = get_qedr_ah(ibah);
2889
2890	rdma_destroy_ah_attr(ah_attr: &ah->attr);
2891	return 0;
2892	}
2893
2894	static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2895	{
2896	struct qedr_pbl *pbl, *tmp;
2897
2898	if (info->pbl_table)
2899	list_add_tail(new: &info->pbl_table->list_entry,
2900	head: &info->free_pbl_list);
2901
2902	if (!list_empty(head: &info->inuse_pbl_list))
2903	list_splice(list: &info->inuse_pbl_list, head: &info->free_pbl_list);
2904
2905	list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2906	list_del(entry: &pbl->list_entry);
2907	qedr_free_pbl(dev, pbl_info: &info->pbl_info, pbl);
2908	}
2909	}
2910
2911	static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2912	size_t page_list_len, bool two_layered)
2913	{
2914	struct qedr_pbl *tmp;
2915	int rc;
2916
2917	INIT_LIST_HEAD(list: &info->free_pbl_list);
2918	INIT_LIST_HEAD(list: &info->inuse_pbl_list);
2919
2920	rc = qedr_prepare_pbl_tbl(dev, pbl_info: &info->pbl_info,
2921	num_pbes: page_list_len, two_layer_capable: two_layered);
2922	if (rc)
2923	goto done;
2924
2925	info->pbl_table = qedr_alloc_pbl_tbl(dev, pbl_info: &info->pbl_info, GFP_KERNEL);
2926	if (IS_ERR(ptr: info->pbl_table)) {
2927	rc = PTR_ERR(ptr: info->pbl_table);
2928	goto done;
2929	}
2930
2931	DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2932	&info->pbl_table->pa);
2933
2934	/* in usual case we use 2 PBLs, so we add one to free
2935	* list and allocating another one
2936	*/
2937	tmp = qedr_alloc_pbl_tbl(dev, pbl_info: &info->pbl_info, GFP_KERNEL);
2938	if (IS_ERR(ptr: tmp)) {
2939	DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2940	goto done;
2941	}
2942
2943	list_add_tail(new: &tmp->list_entry, head: &info->free_pbl_list);
2944
2945	DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2946
2947	done:
2948	if (rc)
2949	free_mr_info(dev, info);
2950
2951	return rc;
2952	}
2953
2954	struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2955	u64 usr_addr, int acc, struct ib_udata *udata)
2956	{
2957	struct qedr_dev *dev = get_qedr_dev(ibdev: ibpd->device);
2958	struct qedr_mr *mr;
2959	struct qedr_pd *pd;
2960	int rc = -ENOMEM;
2961
2962	pd = get_qedr_pd(ibpd);
2963	DP_DEBUG(dev, QEDR_MSG_MR,
2964	"qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2965	pd->pd_id, start, len, usr_addr, acc);
2966
2967	if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2968	return ERR_PTR(error: -EINVAL);
2969
2970	mr = kzalloc(size: sizeof(*mr), GFP_KERNEL);
2971	if (!mr)
2972	return ERR_PTR(error: rc);
2973
2974	mr->type = QEDR_MR_USER;
2975
2976	mr->umem = ib_umem_get(device: ibpd->device, addr: start, size: len, access: acc);
2977	if (IS_ERR(ptr: mr->umem)) {
2978	rc = -EFAULT;
2979	goto err0;
2980	}
2981
2982	rc = init_mr_info(dev, info: &mr->info,
2983	page_list_len: ib_umem_num_dma_blocks(umem: mr->umem, PAGE_SIZE), two_layered: 1);
2984	if (rc)
2985	goto err1;
2986
2987	qedr_populate_pbls(dev, umem: mr->umem, pbl: mr->info.pbl_table,
2988	pbl_info: &mr->info.pbl_info, PAGE_SHIFT);
2989
2990	rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2991	if (rc) {
2992	if (rc == -EINVAL)
2993	DP_ERR(dev, "Out of MR resources\n");
2994	else
2995	DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
2996
2997	goto err1;
2998	}
2999
3000	/* Index only, 18 bit long, lkey = itid << 8 | key */
3001	mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
3002	mr->hw_mr.key = 0;
3003	mr->hw_mr.pd = pd->pd_id;
3004	mr->hw_mr.local_read = 1;
3005	mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3006	mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3007	mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3008	mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3009	mr->hw_mr.mw_bind = false;
3010	mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
3011	mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
3012	mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
3013	mr->hw_mr.page_size_log = PAGE_SHIFT;
3014	mr->hw_mr.length = len;
3015	mr->hw_mr.vaddr = usr_addr;
3016	mr->hw_mr.phy_mr = false;
3017	mr->hw_mr.dma_mr = false;
3018
3019	rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3020	if (rc) {
3021	DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3022	goto err2;
3023	}
3024
3025	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3026	if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3027	mr->hw_mr.remote_atomic)
3028	mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3029
3030	DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
3031	mr->ibmr.lkey);
3032	return &mr->ibmr;
3033
3034	err2:
3035	dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3036	err1:
3037	qedr_free_pbl(dev, pbl_info: &mr->info.pbl_info, pbl: mr->info.pbl_table);
3038	err0:
3039	kfree(objp: mr);
3040	return ERR_PTR(error: rc);
3041	}
3042
3043	int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
3044	{
3045	struct qedr_mr *mr = get_qedr_mr(ibmr: ib_mr);
3046	struct qedr_dev *dev = get_qedr_dev(ibdev: ib_mr->device);
3047	int rc = 0;
3048
3049	rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
3050	if (rc)
3051	return rc;
3052
3053	dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3054
3055	if (mr->type != QEDR_MR_DMA)
3056	free_mr_info(dev, info: &mr->info);
3057
3058	/* it could be user registered memory. */
3059	ib_umem_release(umem: mr->umem);
3060
3061	kfree(objp: mr);
3062
3063	return rc;
3064	}
3065
3066	static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
3067	int max_page_list_len)
3068	{
3069	struct qedr_pd *pd = get_qedr_pd(ibpd);
3070	struct qedr_dev *dev = get_qedr_dev(ibdev: ibpd->device);
3071	struct qedr_mr *mr;
3072	int rc = -ENOMEM;
3073
3074	DP_DEBUG(dev, QEDR_MSG_MR,
3075	"qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
3076	max_page_list_len);
3077
3078	mr = kzalloc(size: sizeof(*mr), GFP_KERNEL);
3079	if (!mr)
3080	return ERR_PTR(error: rc);
3081
3082	mr->dev = dev;
3083	mr->type = QEDR_MR_FRMR;
3084
3085	rc = init_mr_info(dev, info: &mr->info, page_list_len: max_page_list_len, two_layered: 1);
3086	if (rc)
3087	goto err0;
3088
3089	rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3090	if (rc) {
3091	if (rc == -EINVAL)
3092	DP_ERR(dev, "Out of MR resources\n");
3093	else
3094	DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
3095
3096	goto err1;
3097	}
3098
3099	/* Index only, 18 bit long, lkey = itid << 8 | key */
3100	mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
3101	mr->hw_mr.key = 0;
3102	mr->hw_mr.pd = pd->pd_id;
3103	mr->hw_mr.local_read = 1;
3104	mr->hw_mr.local_write = 0;
3105	mr->hw_mr.remote_read = 0;
3106	mr->hw_mr.remote_write = 0;
3107	mr->hw_mr.remote_atomic = 0;
3108	mr->hw_mr.mw_bind = false;
3109	mr->hw_mr.pbl_ptr = 0;
3110	mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
3111	mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
3112	mr->hw_mr.length = 0;
3113	mr->hw_mr.vaddr = 0;
3114	mr->hw_mr.phy_mr = true;
3115	mr->hw_mr.dma_mr = false;
3116
3117	rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3118	if (rc) {
3119	DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3120	goto err2;
3121	}
3122
3123	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3124	mr->ibmr.rkey = mr->ibmr.lkey;
3125
3126	DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
3127	return mr;
3128
3129	err2:
3130	dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3131	err1:
3132	qedr_free_pbl(dev, pbl_info: &mr->info.pbl_info, pbl: mr->info.pbl_table);
3133	err0:
3134	kfree(objp: mr);
3135	return ERR_PTR(error: rc);
3136	}
3137
3138	struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
3139	u32 max_num_sg)
3140	{
3141	struct qedr_mr *mr;
3142
3143	if (mr_type != IB_MR_TYPE_MEM_REG)
3144	return ERR_PTR(error: -EINVAL);
3145
3146	mr = __qedr_alloc_mr(ibpd, max_page_list_len: max_num_sg);
3147
3148	if (IS_ERR(ptr: mr))
3149	return ERR_PTR(error: -EINVAL);
3150
3151	return &mr->ibmr;
3152	}
3153
3154	static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
3155	{
3156	struct qedr_mr *mr = get_qedr_mr(ibmr);
3157	struct qedr_pbl *pbl_table;
3158	struct regpair *pbe;
3159	u32 pbes_in_page;
3160
3161	if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
3162	DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
3163	return -ENOMEM;
3164	}
3165
3166	DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
3167	mr->npages, addr);
3168
3169	pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
3170	pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
3171	pbe = (struct regpair *)pbl_table->va;
3172	pbe += mr->npages % pbes_in_page;
3173	pbe->lo = cpu_to_le32((u32)addr);
3174	pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
3175
3176	mr->npages++;
3177
3178	return 0;
3179	}
3180
3181	static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
3182	{
3183	int work = info->completed - info->completed_handled - 1;
3184
3185	DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
3186	while (work-- > 0 && !list_empty(head: &info->inuse_pbl_list)) {
3187	struct qedr_pbl *pbl;
3188
3189	/* Free all the page list that are possible to be freed
3190	* (all the ones that were invalidated), under the assumption
3191	* that if an FMR was completed successfully that means that
3192	* if there was an invalidate operation before it also ended
3193	*/
3194	pbl = list_first_entry(&info->inuse_pbl_list,
3195	struct qedr_pbl, list_entry);
3196	list_move_tail(list: &pbl->list_entry, head: &info->free_pbl_list);
3197	info->completed_handled++;
3198	}
3199	}
3200
3201	int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
3202	int sg_nents, unsigned int *sg_offset)
3203	{
3204	struct qedr_mr *mr = get_qedr_mr(ibmr);
3205
3206	mr->npages = 0;
3207
3208	handle_completed_mrs(dev: mr->dev, info: &mr->info);
3209	return ib_sg_to_pages(mr: ibmr, sgl: sg, sg_nents, NULL, set_page: qedr_set_page);
3210	}
3211
3212	struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
3213	{
3214	struct qedr_dev *dev = get_qedr_dev(ibdev: ibpd->device);
3215	struct qedr_pd *pd = get_qedr_pd(ibpd);
3216	struct qedr_mr *mr;
3217	int rc;
3218
3219	mr = kzalloc(size: sizeof(*mr), GFP_KERNEL);
3220	if (!mr)
3221	return ERR_PTR(error: -ENOMEM);
3222
3223	mr->type = QEDR_MR_DMA;
3224
3225	rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3226	if (rc) {
3227	if (rc == -EINVAL)
3228	DP_ERR(dev, "Out of MR resources\n");
3229	else
3230	DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
3231
3232	goto err1;
3233	}
3234
3235	/* index only, 18 bit long, lkey = itid << 8 | key */
3236	mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
3237	mr->hw_mr.pd = pd->pd_id;
3238	mr->hw_mr.local_read = 1;
3239	mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3240	mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3241	mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3242	mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3243	mr->hw_mr.dma_mr = true;
3244
3245	rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3246	if (rc) {
3247	DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3248	goto err2;
3249	}
3250
3251	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3252	if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3253	mr->hw_mr.remote_atomic)
3254	mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3255
3256	DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
3257	return &mr->ibmr;
3258
3259	err2:
3260	dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3261	err1:
3262	kfree(objp: mr);
3263	return ERR_PTR(error: rc);
3264	}
3265
3266	static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
3267	{
3268	return (((wq->prod + 1) % wq->max_wr) == wq->cons);
3269	}
3270
3271	static int sge_data_len(struct ib_sge *sg_list, int num_sge)
3272	{
3273	int i, len = 0;
3274
3275	for (i = 0; i < num_sge; i++)
3276	len += sg_list[i].length;
3277
3278	return len;
3279	}
3280
3281	static void swap_wqe_data64(u64 *p)
3282	{
3283	int i;
3284
3285	for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
3286	*p = cpu_to_be64(cpu_to_le64(*p));
3287	}
3288
3289	static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
3290	struct qedr_qp *qp, u8 *wqe_size,
3291	const struct ib_send_wr *wr,
3292	const struct ib_send_wr **bad_wr,
3293	u8 *bits, u8 bit)
3294	{
3295	u32 data_size = sge_data_len(sg_list: wr->sg_list, num_sge: wr->num_sge);
3296	char *seg_prt, *wqe;
3297	int i, seg_siz;
3298
3299	if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
3300	DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
3301	*bad_wr = wr;
3302	return 0;
3303	}
3304
3305	if (!data_size)
3306	return data_size;
3307
3308	*bits |= bit;
3309
3310	seg_prt = NULL;
3311	wqe = NULL;
3312	seg_siz = 0;
3313
3314	/* Copy data inline */
3315	for (i = 0; i < wr->num_sge; i++) {
3316	u32 len = wr->sg_list[i].length;
3317	void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
3318
3319	while (len > 0) {
3320	u32 cur;
3321
3322	/* New segment required */
3323	if (!seg_siz) {
3324	wqe = (char *)qed_chain_produce(p_chain: &qp->sq.pbl);
3325	seg_prt = wqe;
3326	seg_siz = sizeof(struct rdma_sq_common_wqe);
3327	(*wqe_size)++;
3328	}
3329
3330	/* Calculate currently allowed length */
3331	cur = min_t(u32, len, seg_siz);
3332	memcpy(seg_prt, src, cur);
3333
3334	/* Update segment variables */
3335	seg_prt += cur;
3336	seg_siz -= cur;
3337
3338	/* Update sge variables */
3339	src += cur;
3340	len -= cur;
3341
3342	/* Swap fully-completed segments */
3343	if (!seg_siz)
3344	swap_wqe_data64(p: (u64 *)wqe);
3345	}
3346	}
3347
3348	/* swap last not completed segment */
3349	if (seg_siz)
3350	swap_wqe_data64(p: (u64 *)wqe);
3351
3352	return data_size;
3353	}
3354
3355	#define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
3356	do { \
3357	DMA_REGPAIR_LE(sge->addr, vaddr); \
3358	(sge)->length = cpu_to_le32(vlength); \
3359	(sge)->flags = cpu_to_le32(vflags); \
3360	} while (0)
3361
3362	#define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
3363	do { \
3364	DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \
3365	(hdr)->num_sges = num_sge; \
3366	} while (0)
3367
3368	#define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
3369	do { \
3370	DMA_REGPAIR_LE(sge->addr, vaddr); \
3371	(sge)->length = cpu_to_le32(vlength); \
3372	(sge)->l_key = cpu_to_le32(vlkey); \
3373	} while (0)
3374
3375	static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
3376	const struct ib_send_wr *wr)
3377	{
3378	u32 data_size = 0;
3379	int i;
3380
3381	for (i = 0; i < wr->num_sge; i++) {
3382	struct rdma_sq_sge *sge = qed_chain_produce(p_chain: &qp->sq.pbl);
3383
3384	DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
3385	sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3386	sge->length = cpu_to_le32(wr->sg_list[i].length);
3387	data_size += wr->sg_list[i].length;
3388	}
3389
3390	if (wqe_size)
3391	*wqe_size += wr->num_sge;
3392
3393	return data_size;
3394	}
3395
3396	static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3397	struct qedr_qp *qp,
3398	struct rdma_sq_rdma_wqe_1st *rwqe,
3399	struct rdma_sq_rdma_wqe_2nd *rwqe2,
3400	const struct ib_send_wr *wr,
3401	const struct ib_send_wr **bad_wr)
3402	{
3403	rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3404	DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3405
3406	if (wr->send_flags & IB_SEND_INLINE &&
3407	(wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3408	wr->opcode == IB_WR_RDMA_WRITE)) {
3409	u8 flags = 0;
3410
3411	SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3412	return qedr_prepare_sq_inline_data(dev, qp, wqe_size: &rwqe->wqe_size, wr,
3413	bad_wr, bits: &rwqe->flags, bit: flags);
3414	}
3415
3416	return qedr_prepare_sq_sges(qp, wqe_size: &rwqe->wqe_size, wr);
3417	}
3418
3419	static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3420	struct qedr_qp *qp,
3421	struct rdma_sq_send_wqe_1st *swqe,
3422	struct rdma_sq_send_wqe_2st *swqe2,
3423	const struct ib_send_wr *wr,
3424	const struct ib_send_wr **bad_wr)
3425	{
3426	memset(swqe2, 0, sizeof(*swqe2));
3427	if (wr->send_flags & IB_SEND_INLINE) {
3428	u8 flags = 0;
3429
3430	SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3431	return qedr_prepare_sq_inline_data(dev, qp, wqe_size: &swqe->wqe_size, wr,
3432	bad_wr, bits: &swqe->flags, bit: flags);
3433	}
3434
3435	return qedr_prepare_sq_sges(qp, wqe_size: &swqe->wqe_size, wr);
3436	}
3437
3438	static int qedr_prepare_reg(struct qedr_qp *qp,
3439	struct rdma_sq_fmr_wqe_1st *fwqe1,
3440	const struct ib_reg_wr *wr)
3441	{
3442	struct qedr_mr *mr = get_qedr_mr(ibmr: wr->mr);
3443	struct rdma_sq_fmr_wqe_2nd *fwqe2;
3444
3445	fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(p_chain: &qp->sq.pbl);
3446	fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3447	fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3448	fwqe1->l_key = wr->key;
3449
3450	fwqe2->access_ctrl = 0;
3451
3452	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3453	!!(wr->access & IB_ACCESS_REMOTE_READ));
3454	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3455	!!(wr->access & IB_ACCESS_REMOTE_WRITE));
3456	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3457	!!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3458	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3459	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3460	!!(wr->access & IB_ACCESS_LOCAL_WRITE));
3461	fwqe2->fmr_ctrl = 0;
3462
3463	SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3464	ilog2(mr->ibmr.page_size) - 12);
3465
3466	fwqe2->length_hi = 0;
3467	fwqe2->length_lo = mr->ibmr.length;
3468	fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3469	fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3470
3471	qp->wqe_wr_id[qp->sq.prod].mr = mr;
3472
3473	return 0;
3474	}
3475
3476	static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
3477	{
3478	switch (opcode) {
3479	case IB_WR_RDMA_WRITE:
3480	case IB_WR_RDMA_WRITE_WITH_IMM:
3481	return IB_WC_RDMA_WRITE;
3482	case IB_WR_SEND_WITH_IMM:
3483	case IB_WR_SEND:
3484	case IB_WR_SEND_WITH_INV:
3485	return IB_WC_SEND;
3486	case IB_WR_RDMA_READ:
3487	case IB_WR_RDMA_READ_WITH_INV:
3488	return IB_WC_RDMA_READ;
3489	case IB_WR_ATOMIC_CMP_AND_SWP:
3490	return IB_WC_COMP_SWAP;
3491	case IB_WR_ATOMIC_FETCH_AND_ADD:
3492	return IB_WC_FETCH_ADD;
3493	case IB_WR_REG_MR:
3494	return IB_WC_REG_MR;
3495	case IB_WR_LOCAL_INV:
3496	return IB_WC_LOCAL_INV;
3497	default:
3498	return IB_WC_SEND;
3499	}
3500	}
3501
3502	static inline bool qedr_can_post_send(struct qedr_qp *qp,
3503	const struct ib_send_wr *wr)
3504	{
3505	int wq_is_full, err_wr, pbl_is_full;
3506	struct qedr_dev *dev = qp->dev;
3507
3508	/* prevent SQ overflow and/or processing of a bad WR */
3509	err_wr = wr->num_sge > qp->sq.max_sges;
3510	wq_is_full = qedr_wq_is_full(wq: &qp->sq);
3511	pbl_is_full = qed_chain_get_elem_left_u32(chain: &qp->sq.pbl) <
3512	QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3513	if (wq_is_full || err_wr || pbl_is_full) {
3514	if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3515	DP_ERR(dev,
3516	"error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3517	qp);
3518	qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3519	}
3520
3521	if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3522	DP_ERR(dev,
3523	"error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3524	qp);
3525	qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3526	}
3527
3528	if (pbl_is_full &&
3529	!(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3530	DP_ERR(dev,
3531	"error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3532	qp);
3533	qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3534	}
3535	return false;
3536	}
3537	return true;
3538	}
3539
3540	static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3541	const struct ib_send_wr **bad_wr)
3542	{
3543	struct qedr_dev *dev = get_qedr_dev(ibdev: ibqp->device);
3544	struct qedr_qp *qp = get_qedr_qp(ibqp);
3545	struct rdma_sq_atomic_wqe_1st *awqe1;
3546	struct rdma_sq_atomic_wqe_2nd *awqe2;
3547	struct rdma_sq_atomic_wqe_3rd *awqe3;
3548	struct rdma_sq_send_wqe_2st *swqe2;
3549	struct rdma_sq_local_inv_wqe *iwqe;
3550	struct rdma_sq_rdma_wqe_2nd *rwqe2;
3551	struct rdma_sq_send_wqe_1st *swqe;
3552	struct rdma_sq_rdma_wqe_1st *rwqe;
3553	struct rdma_sq_fmr_wqe_1st *fwqe1;
3554	struct rdma_sq_common_wqe *wqe;
3555	u32 length;
3556	int rc = 0;
3557	bool comp;
3558
3559	if (!qedr_can_post_send(qp, wr)) {
3560	*bad_wr = wr;
3561	return -ENOMEM;
3562	}
3563
3564	wqe = qed_chain_produce(p_chain: &qp->sq.pbl);
3565	qp->wqe_wr_id[qp->sq.prod].signaled =
3566	!!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3567
3568	wqe->flags = 0;
3569	SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3570	!!(wr->send_flags & IB_SEND_SOLICITED));
3571	comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3572	SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3573	SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3574	!!(wr->send_flags & IB_SEND_FENCE));
3575	wqe->prev_wqe_size = qp->prev_wqe_size;
3576
3577	qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(opcode: wr->opcode);
3578
3579	switch (wr->opcode) {
3580	case IB_WR_SEND_WITH_IMM:
3581	if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3582	rc = -EINVAL;
3583	*bad_wr = wr;
3584	break;
3585	}
3586	wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3587	swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3588	swqe->wqe_size = 2;
3589	swqe2 = qed_chain_produce(p_chain: &qp->sq.pbl);
3590
3591	swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
3592	length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3593	wr, bad_wr);
3594	swqe->length = cpu_to_le32(length);
3595	qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3596	qp->prev_wqe_size = swqe->wqe_size;
3597	qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3598	break;
3599	case IB_WR_SEND:
3600	wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3601	swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3602
3603	swqe->wqe_size = 2;
3604	swqe2 = qed_chain_produce(p_chain: &qp->sq.pbl);
3605	length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3606	wr, bad_wr);
3607	swqe->length = cpu_to_le32(length);
3608	qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3609	qp->prev_wqe_size = swqe->wqe_size;
3610	qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3611	break;
3612	case IB_WR_SEND_WITH_INV:
3613	wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3614	swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3615	swqe2 = qed_chain_produce(p_chain: &qp->sq.pbl);
3616	swqe->wqe_size = 2;
3617	swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3618	length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3619	wr, bad_wr);
3620	swqe->length = cpu_to_le32(length);
3621	qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3622	qp->prev_wqe_size = swqe->wqe_size;
3623	qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3624	break;
3625
3626	case IB_WR_RDMA_WRITE_WITH_IMM:
3627	if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3628	rc = -EINVAL;
3629	*bad_wr = wr;
3630	break;
3631	}
3632	wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3633	rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3634
3635	rwqe->wqe_size = 2;
3636	rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3637	rwqe2 = qed_chain_produce(p_chain: &qp->sq.pbl);
3638	length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3639	wr, bad_wr);
3640	rwqe->length = cpu_to_le32(length);
3641	qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3642	qp->prev_wqe_size = rwqe->wqe_size;
3643	qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3644	break;
3645	case IB_WR_RDMA_WRITE:
3646	wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3647	rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3648
3649	rwqe->wqe_size = 2;
3650	rwqe2 = qed_chain_produce(p_chain: &qp->sq.pbl);
3651	length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3652	wr, bad_wr);
3653	rwqe->length = cpu_to_le32(length);
3654	qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3655	qp->prev_wqe_size = rwqe->wqe_size;
3656	qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3657	break;
3658	case IB_WR_RDMA_READ_WITH_INV:
3659	SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
3660	fallthrough; /* same is identical to RDMA READ */
3661
3662	case IB_WR_RDMA_READ:
3663	wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3664	rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3665
3666	rwqe->wqe_size = 2;
3667	rwqe2 = qed_chain_produce(p_chain: &qp->sq.pbl);
3668	length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3669	wr, bad_wr);
3670	rwqe->length = cpu_to_le32(length);
3671	qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3672	qp->prev_wqe_size = rwqe->wqe_size;
3673	qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3674	break;
3675
3676	case IB_WR_ATOMIC_CMP_AND_SWP:
3677	case IB_WR_ATOMIC_FETCH_AND_ADD:
3678	awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3679	awqe1->wqe_size = 4;
3680
3681	awqe2 = qed_chain_produce(p_chain: &qp->sq.pbl);
3682	DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3683	awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3684
3685	awqe3 = qed_chain_produce(p_chain: &qp->sq.pbl);
3686
3687	if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3688	wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3689	DMA_REGPAIR_LE(awqe3->swap_data,
3690	atomic_wr(wr)->compare_add);
3691	} else {
3692	wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3693	DMA_REGPAIR_LE(awqe3->swap_data,
3694	atomic_wr(wr)->swap);
3695	DMA_REGPAIR_LE(awqe3->cmp_data,
3696	atomic_wr(wr)->compare_add);
3697	}
3698
3699	qedr_prepare_sq_sges(qp, NULL, wr);
3700
3701	qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3702	qp->prev_wqe_size = awqe1->wqe_size;
3703	break;
3704
3705	case IB_WR_LOCAL_INV:
3706	iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3707	iwqe->wqe_size = 1;
3708
3709	iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3710	iwqe->inv_l_key = wr->ex.invalidate_rkey;
3711	qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3712	qp->prev_wqe_size = iwqe->wqe_size;
3713	break;
3714	case IB_WR_REG_MR:
3715	DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3716	wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3717	fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3718	fwqe1->wqe_size = 2;
3719
3720	rc = qedr_prepare_reg(qp, fwqe1, wr: reg_wr(wr));
3721	if (rc) {
3722	DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3723	*bad_wr = wr;
3724	break;
3725	}
3726
3727	qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3728	qp->prev_wqe_size = fwqe1->wqe_size;
3729	break;
3730	default:
3731	DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3732	rc = -EINVAL;
3733	*bad_wr = wr;
3734	break;
3735	}
3736
3737	if (*bad_wr) {
3738	u16 value;
3739
3740	/* Restore prod to its position before
3741	* this WR was processed
3742	*/
3743	value = le16_to_cpu(qp->sq.db_data.data.value);
3744	qed_chain_set_prod(p_chain: &qp->sq.pbl, prod_idx: value, p_prod_elem: wqe);
3745
3746	/* Restore prev_wqe_size */
3747	qp->prev_wqe_size = wqe->prev_wqe_size;
3748	rc = -EINVAL;
3749	DP_ERR(dev, "POST SEND FAILED\n");
3750	}
3751
3752	return rc;
3753	}
3754
3755	int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3756	const struct ib_send_wr **bad_wr)
3757	{
3758	struct qedr_dev *dev = get_qedr_dev(ibdev: ibqp->device);
3759	struct qedr_qp *qp = get_qedr_qp(ibqp);
3760	unsigned long flags;
3761	int rc = 0;
3762
3763	*bad_wr = NULL;
3764
3765	if (qp->qp_type == IB_QPT_GSI)
3766	return qedr_gsi_post_send(ibqp, wr, bad_wr);
3767
3768	spin_lock_irqsave(&qp->q_lock, flags);
3769
3770	if (rdma_protocol_roce(device: &dev->ibdev, port_num: 1)) {
3771	if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3772	(qp->state != QED_ROCE_QP_STATE_ERR) &&
3773	(qp->state != QED_ROCE_QP_STATE_SQD)) {
3774	spin_unlock_irqrestore(lock: &qp->q_lock, flags);
3775	*bad_wr = wr;
3776	DP_DEBUG(dev, QEDR_MSG_CQ,
3777	"QP in wrong state! QP icid=0x%x state %d\n",
3778	qp->icid, qp->state);
3779	return -EINVAL;
3780	}
3781	}
3782
3783	while (wr) {
3784	rc = __qedr_post_send(ibqp, wr, bad_wr);
3785	if (rc)
3786	break;
3787
3788	qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3789
3790	qedr_inc_sw_prod(info: &qp->sq);
3791
3792	qp->sq.db_data.data.value++;
3793
3794	wr = wr->next;
3795	}
3796
3797	/* Trigger doorbell
3798	* If there was a failure in the first WR then it will be triggered in
3799	* vane. However this is not harmful (as long as the producer value is
3800	* unchanged). For performance reasons we avoid checking for this
3801	* redundant doorbell.
3802	*
3803	* qp->wqe_wr_id is accessed during qedr_poll_cq, as
3804	* soon as we give the doorbell, we could get a completion
3805	* for this wr, therefore we need to make sure that the
3806	* memory is updated before giving the doorbell.
3807	* During qedr_poll_cq, rmb is called before accessing the
3808	* cqe. This covers for the smp_rmb as well.
3809	*/
3810	smp_wmb();
3811	writel(val: qp->sq.db_data.raw, addr: qp->sq.db);
3812
3813	spin_unlock_irqrestore(lock: &qp->q_lock, flags);
3814
3815	return rc;
3816	}
3817
3818	static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3819	{
3820	u32 used;
3821
3822	/* Calculate number of elements used based on producer
3823	* count and consumer count and subtract it from max
3824	* work request supported so that we get elements left.
3825	*/
3826	used = hw_srq->wr_prod_cnt - (u32)atomic_read(v: &hw_srq->wr_cons_cnt);
3827
3828	return hw_srq->max_wr - used;
3829	}
3830
3831	int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3832	const struct ib_recv_wr **bad_wr)
3833	{
3834	struct qedr_srq *srq = get_qedr_srq(ibsrq);
3835	struct qedr_srq_hwq_info *hw_srq;
3836	struct qedr_dev *dev = srq->dev;
3837	struct qed_chain *pbl;
3838	unsigned long flags;
3839	int status = 0;
3840	u32 num_sge;
3841
3842	spin_lock_irqsave(&srq->lock, flags);
3843
3844	hw_srq = &srq->hw_srq;
3845	pbl = &srq->hw_srq.pbl;
3846	while (wr) {
3847	struct rdma_srq_wqe_header *hdr;
3848	int i;
3849
3850	if (!qedr_srq_elem_left(hw_srq) ||
3851	wr->num_sge > srq->hw_srq.max_sges) {
3852	DP_ERR(dev, "Can't post WR (%d,%d) || (%d > %d)\n",
3853	hw_srq->wr_prod_cnt,
3854	atomic_read(&hw_srq->wr_cons_cnt),
3855	wr->num_sge, srq->hw_srq.max_sges);
3856	status = -ENOMEM;
3857	*bad_wr = wr;
3858	break;
3859	}
3860
3861	hdr = qed_chain_produce(p_chain: pbl);
3862	num_sge = wr->num_sge;
3863	/* Set number of sge and work request id in header */
3864	SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3865
3866	srq->hw_srq.wr_prod_cnt++;
3867	hw_srq->wqe_prod++;
3868	hw_srq->sge_prod++;
3869
3870	DP_DEBUG(dev, QEDR_MSG_SRQ,
3871	"SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3872	wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3873
3874	for (i = 0; i < wr->num_sge; i++) {
3875	struct rdma_srq_sge *srq_sge = qed_chain_produce(p_chain: pbl);
3876
3877	/* Set SGE length, lkey and address */
3878	SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3879	wr->sg_list[i].length, wr->sg_list[i].lkey);
3880
3881	DP_DEBUG(dev, QEDR_MSG_SRQ,
3882	"[%d]: len %d key %x addr %x:%x\n",
3883	i, srq_sge->length, srq_sge->l_key,
3884	srq_sge->addr.hi, srq_sge->addr.lo);
3885	hw_srq->sge_prod++;
3886	}
3887
3888	/* Update WQE and SGE information before
3889	* updating producer.
3890	*/
3891	dma_wmb();
3892
3893	/* SRQ producer is 8 bytes. Need to update SGE producer index
3894	* in first 4 bytes and need to update WQE producer in
3895	* next 4 bytes.
3896	*/
3897	srq->hw_srq.virt_prod_pair_addr->sge_prod = cpu_to_le32(hw_srq->sge_prod);
3898	/* Make sure sge producer is updated first */
3899	dma_wmb();
3900	srq->hw_srq.virt_prod_pair_addr->wqe_prod = cpu_to_le32(hw_srq->wqe_prod);
3901
3902	wr = wr->next;
3903	}
3904
3905	DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3906	qed_chain_get_elem_left(pbl));
3907	spin_unlock_irqrestore(lock: &srq->lock, flags);
3908
3909	return status;
3910	}
3911
3912	int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3913	const struct ib_recv_wr **bad_wr)
3914	{
3915	struct qedr_qp *qp = get_qedr_qp(ibqp);
3916	struct qedr_dev *dev = qp->dev;
3917	unsigned long flags;
3918	int status = 0;
3919
3920	if (qp->qp_type == IB_QPT_GSI)
3921	return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3922
3923	spin_lock_irqsave(&qp->q_lock, flags);
3924
3925	while (wr) {
3926	int i;
3927
3928	if (qed_chain_get_elem_left_u32(chain: &qp->rq.pbl) <
3929	QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3930	wr->num_sge > qp->rq.max_sges) {
3931	DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n",
3932	qed_chain_get_elem_left_u32(&qp->rq.pbl),
3933	QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3934	qp->rq.max_sges);
3935	status = -ENOMEM;
3936	*bad_wr = wr;
3937	break;
3938	}
3939	for (i = 0; i < wr->num_sge; i++) {
3940	u32 flags = 0;
3941	struct rdma_rq_sge *rqe =
3942	qed_chain_produce(p_chain: &qp->rq.pbl);
3943
3944	/* First one must include the number
3945	* of SGE in the list
3946	*/
3947	if (!i)
3948	SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3949	wr->num_sge);
3950
3951	SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
3952	wr->sg_list[i].lkey);
3953
3954	RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3955	wr->sg_list[i].length, flags);
3956	}
3957
3958	/* Special case of no sges. FW requires between 1-4 sges...
3959	* in this case we need to post 1 sge with length zero. this is
3960	* because rdma write with immediate consumes an RQ.
3961	*/
3962	if (!wr->num_sge) {
3963	u32 flags = 0;
3964	struct rdma_rq_sge *rqe =
3965	qed_chain_produce(p_chain: &qp->rq.pbl);
3966
3967	/* First one must include the number
3968	* of SGE in the list
3969	*/
3970	SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
3971	SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3972
3973	RQ_SGE_SET(rqe, 0, 0, flags);
3974	i = 1;
3975	}
3976
3977	qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3978	qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3979
3980	qedr_inc_sw_prod(info: &qp->rq);
3981
3982	/* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3983	* soon as we give the doorbell, we could get a completion
3984	* for this wr, therefore we need to make sure that the
3985	* memory is update before giving the doorbell.
3986	* During qedr_poll_cq, rmb is called before accessing the
3987	* cqe. This covers for the smp_rmb as well.
3988	*/
3989	smp_wmb();
3990
3991	qp->rq.db_data.data.value++;
3992
3993	writel(val: qp->rq.db_data.raw, addr: qp->rq.db);
3994
3995	if (rdma_protocol_iwarp(device: &dev->ibdev, port_num: 1)) {
3996	writel(val: qp->rq.iwarp_db2_data.raw, addr: qp->rq.iwarp_db2);
3997	}
3998
3999	wr = wr->next;
4000	}
4001
4002	spin_unlock_irqrestore(lock: &qp->q_lock, flags);
4003
4004	return status;
4005	}
4006
4007	static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
4008	{
4009	struct rdma_cqe_requester *resp_cqe = &cqe->req;
4010
4011	return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
4012	cq->pbl_toggle;
4013	}
4014
4015	static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
4016	{
4017	struct rdma_cqe_requester *resp_cqe = &cqe->req;
4018	struct qedr_qp *qp;
4019
4020	qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
4021	resp_cqe->qp_handle.lo,
4022	u64);
4023	return qp;
4024	}
4025
4026	static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
4027	{
4028	struct rdma_cqe_requester *resp_cqe = &cqe->req;
4029
4030	return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
4031	}
4032
4033	/* Return latest CQE (needs processing) */
4034	static union rdma_cqe *get_cqe(struct qedr_cq *cq)
4035	{
4036	return cq->latest_cqe;
4037	}
4038
4039	/* In fmr we need to increase the number of fmr completed counter for the fmr
4040	* algorithm determining whether we can free a pbl or not.
4041	* we need to perform this whether the work request was signaled or not. for
4042	* this purpose we call this function from the condition that checks if a wr
4043	* should be skipped, to make sure we don't miss it ( possibly this fmr
4044	* operation was not signalted)
4045	*/
4046	static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
4047	{
4048	if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
4049	qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
4050	}
4051
4052	static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
4053	struct qedr_cq *cq, int num_entries,
4054	struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
4055	int force)
4056	{
4057	u16 cnt = 0;
4058
4059	while (num_entries && qp->sq.wqe_cons != hw_cons) {
4060	if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
4061	qedr_chk_if_fmr(qp);
4062	/* skip WC */
4063	goto next_cqe;
4064	}
4065
4066	/* fill WC */
4067	wc->status = status;
4068	wc->vendor_err = 0;
4069	wc->wc_flags = 0;
4070	wc->src_qp = qp->id;
4071	wc->qp = &qp->ibqp;
4072
4073	wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
4074	wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
4075
4076	switch (wc->opcode) {
4077	case IB_WC_RDMA_WRITE:
4078	wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
4079	break;
4080	case IB_WC_COMP_SWAP:
4081	case IB_WC_FETCH_ADD:
4082	wc->byte_len = 8;
4083	break;
4084	case IB_WC_REG_MR:
4085	qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
4086	break;
4087	case IB_WC_RDMA_READ:
4088	case IB_WC_SEND:
4089	wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
4090	break;
4091	default:
4092	break;
4093	}
4094
4095	num_entries--;
4096	wc++;
4097	cnt++;
4098	next_cqe:
4099	while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
4100	qed_chain_consume(p_chain: &qp->sq.pbl);
4101	qedr_inc_sw_cons(info: &qp->sq);
4102	}
4103
4104	return cnt;
4105	}
4106
4107	static int qedr_poll_cq_req(struct qedr_dev *dev,
4108	struct qedr_qp *qp, struct qedr_cq *cq,
4109	int num_entries, struct ib_wc *wc,
4110	struct rdma_cqe_requester *req)
4111	{
4112	int cnt = 0;
4113
4114	switch (req->status) {
4115	case RDMA_CQE_REQ_STS_OK:
4116	cnt = process_req(dev, qp, cq, num_entries, wc, hw_cons: req->sq_cons,
4117	status: IB_WC_SUCCESS, force: 0);
4118	break;
4119	case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
4120	if (qp->state != QED_ROCE_QP_STATE_ERR)
4121	DP_DEBUG(dev, QEDR_MSG_CQ,
4122	"Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4123	cq->icid, qp->icid);
4124	cnt = process_req(dev, qp, cq, num_entries, wc, hw_cons: req->sq_cons,
4125	status: IB_WC_WR_FLUSH_ERR, force: 1);
4126	break;
4127	default:
4128	/* process all WQE before the cosumer */
4129	qp->state = QED_ROCE_QP_STATE_ERR;
4130	cnt = process_req(dev, qp, cq, num_entries, wc,
4131	hw_cons: req->sq_cons - 1, status: IB_WC_SUCCESS, force: 0);
4132	wc += cnt;
4133	/* if we have extra WC fill it with actual error info */
4134	if (cnt < num_entries) {
4135	enum ib_wc_status wc_status;
4136
4137	switch (req->status) {
4138	case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
4139	DP_ERR(dev,
4140	"Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4141	cq->icid, qp->icid);
4142	wc_status = IB_WC_BAD_RESP_ERR;
4143	break;
4144	case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
4145	DP_ERR(dev,
4146	"Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4147	cq->icid, qp->icid);
4148	wc_status = IB_WC_LOC_LEN_ERR;
4149	break;
4150	case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
4151	DP_ERR(dev,
4152	"Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4153	cq->icid, qp->icid);
4154	wc_status = IB_WC_LOC_QP_OP_ERR;
4155	break;
4156	case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
4157	DP_ERR(dev,
4158	"Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4159	cq->icid, qp->icid);
4160	wc_status = IB_WC_LOC_PROT_ERR;
4161	break;
4162	case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
4163	DP_ERR(dev,
4164	"Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4165	cq->icid, qp->icid);
4166	wc_status = IB_WC_MW_BIND_ERR;
4167	break;
4168	case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
4169	DP_ERR(dev,
4170	"Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4171	cq->icid, qp->icid);
4172	wc_status = IB_WC_REM_INV_REQ_ERR;
4173	break;
4174	case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
4175	DP_ERR(dev,
4176	"Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4177	cq->icid, qp->icid);
4178	wc_status = IB_WC_REM_ACCESS_ERR;
4179	break;
4180	case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
4181	DP_ERR(dev,
4182	"Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4183	cq->icid, qp->icid);
4184	wc_status = IB_WC_REM_OP_ERR;
4185	break;
4186	case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
4187	DP_ERR(dev,
4188	"Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4189	cq->icid, qp->icid);
4190	wc_status = IB_WC_RNR_RETRY_EXC_ERR;
4191	break;
4192	case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
4193	DP_ERR(dev,
4194	"Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4195	cq->icid, qp->icid);
4196	wc_status = IB_WC_RETRY_EXC_ERR;
4197	break;
4198	default:
4199	DP_ERR(dev,
4200	"Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4201	cq->icid, qp->icid);
4202	wc_status = IB_WC_GENERAL_ERR;
4203	}
4204	cnt += process_req(dev, qp, cq, num_entries: 1, wc, hw_cons: req->sq_cons,
4205	status: wc_status, force: 1);
4206	}
4207	}
4208
4209	return cnt;
4210	}
4211
4212	static inline int qedr_cqe_resp_status_to_ib(u8 status)
4213	{
4214	switch (status) {
4215	case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
4216	return IB_WC_LOC_ACCESS_ERR;
4217	case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
4218	return IB_WC_LOC_LEN_ERR;
4219	case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
4220	return IB_WC_LOC_QP_OP_ERR;
4221	case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
4222	return IB_WC_LOC_PROT_ERR;
4223	case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
4224	return IB_WC_MW_BIND_ERR;
4225	case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
4226	return IB_WC_REM_INV_RD_REQ_ERR;
4227	case RDMA_CQE_RESP_STS_OK:
4228	return IB_WC_SUCCESS;
4229	default:
4230	return IB_WC_GENERAL_ERR;
4231	}
4232	}
4233
4234	static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
4235	struct ib_wc *wc)
4236	{
4237	wc->status = IB_WC_SUCCESS;
4238	wc->byte_len = le32_to_cpu(resp->length);
4239
4240	if (resp->flags & QEDR_RESP_IMM) {
4241	wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
4242	wc->wc_flags |= IB_WC_WITH_IMM;
4243
4244	if (resp->flags & QEDR_RESP_RDMA)
4245	wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
4246
4247	if (resp->flags & QEDR_RESP_INV)
4248	return -EINVAL;
4249
4250	} else if (resp->flags & QEDR_RESP_INV) {
4251	wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
4252	wc->wc_flags |= IB_WC_WITH_INVALIDATE;
4253
4254	if (resp->flags & QEDR_RESP_RDMA)
4255	return -EINVAL;
4256
4257	} else if (resp->flags & QEDR_RESP_RDMA) {
4258	return -EINVAL;
4259	}
4260
4261	return 0;
4262	}
4263
4264	static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4265	struct qedr_cq *cq, struct ib_wc *wc,
4266	struct rdma_cqe_responder *resp, u64 wr_id)
4267	{
4268	/* Must fill fields before qedr_set_ok_cqe_resp_wc() */
4269	wc->opcode = IB_WC_RECV;
4270	wc->wc_flags = 0;
4271
4272	if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
4273	if (qedr_set_ok_cqe_resp_wc(resp, wc))
4274	DP_ERR(dev,
4275	"CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
4276	cq, cq->icid, resp->flags);
4277
4278	} else {
4279	wc->status = qedr_cqe_resp_status_to_ib(status: resp->status);
4280	if (wc->status == IB_WC_GENERAL_ERR)
4281	DP_ERR(dev,
4282	"CQ %p (icid=%d) contains an invalid CQE status %d\n",
4283	cq, cq->icid, resp->status);
4284	}
4285
4286	/* Fill the rest of the WC */
4287	wc->vendor_err = 0;
4288	wc->src_qp = qp->id;
4289	wc->qp = &qp->ibqp;
4290	wc->wr_id = wr_id;
4291	}
4292
4293	static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4294	struct qedr_cq *cq, struct ib_wc *wc,
4295	struct rdma_cqe_responder *resp)
4296	{
4297	struct qedr_srq *srq = qp->srq;
4298	u64 wr_id;
4299
4300	wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
4301	le32_to_cpu(resp->srq_wr_id.lo), u64);
4302
4303	if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4304	wc->status = IB_WC_WR_FLUSH_ERR;
4305	wc->vendor_err = 0;
4306	wc->wr_id = wr_id;
4307	wc->byte_len = 0;
4308	wc->src_qp = qp->id;
4309	wc->qp = &qp->ibqp;
4310	wc->wr_id = wr_id;
4311	} else {
4312	__process_resp_one(dev, qp, cq, wc, resp, wr_id);
4313	}
4314	atomic_inc(v: &srq->hw_srq.wr_cons_cnt);
4315
4316	return 1;
4317	}
4318	static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4319	struct qedr_cq *cq, struct ib_wc *wc,
4320	struct rdma_cqe_responder *resp)
4321	{
4322	u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4323
4324	__process_resp_one(dev, qp, cq, wc, resp, wr_id);
4325
4326	while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4327	qed_chain_consume(p_chain: &qp->rq.pbl);
4328	qedr_inc_sw_cons(info: &qp->rq);
4329
4330	return 1;
4331	}
4332
4333	static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
4334	int num_entries, struct ib_wc *wc, u16 hw_cons)
4335	{
4336	u16 cnt = 0;
4337
4338	while (num_entries && qp->rq.wqe_cons != hw_cons) {
4339	/* fill WC */
4340	wc->status = IB_WC_WR_FLUSH_ERR;
4341	wc->vendor_err = 0;
4342	wc->wc_flags = 0;
4343	wc->src_qp = qp->id;
4344	wc->byte_len = 0;
4345	wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4346	wc->qp = &qp->ibqp;
4347	num_entries--;
4348	wc++;
4349	cnt++;
4350	while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4351	qed_chain_consume(p_chain: &qp->rq.pbl);
4352	qedr_inc_sw_cons(info: &qp->rq);
4353	}
4354
4355	return cnt;
4356	}
4357
4358	static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4359	struct rdma_cqe_responder *resp, int *update)
4360	{
4361	if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
4362	consume_cqe(cq);
4363	*update |= 1;
4364	}
4365	}
4366
4367	static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4368	struct qedr_cq *cq, int num_entries,
4369	struct ib_wc *wc,
4370	struct rdma_cqe_responder *resp)
4371	{
4372	int cnt;
4373
4374	cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4375	consume_cqe(cq);
4376
4377	return cnt;
4378	}
4379
4380	static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4381	struct qedr_cq *cq, int num_entries,
4382	struct ib_wc *wc, struct rdma_cqe_responder *resp,
4383	int *update)
4384	{
4385	int cnt;
4386
4387	if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4388	cnt = process_resp_flush(qp, cq, num_entries, wc,
4389	hw_cons: resp->rq_cons_or_srq_id);
4390	try_consume_resp_cqe(cq, qp, resp, update);
4391	} else {
4392	cnt = process_resp_one(dev, qp, cq, wc, resp);
4393	consume_cqe(cq);
4394	*update |= 1;
4395	}
4396
4397	return cnt;
4398	}
4399
4400	static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4401	struct rdma_cqe_requester *req, int *update)
4402	{
4403	if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4404	consume_cqe(cq);
4405	*update |= 1;
4406	}
4407	}
4408
4409	int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4410	{
4411	struct qedr_dev *dev = get_qedr_dev(ibdev: ibcq->device);
4412	struct qedr_cq *cq = get_qedr_cq(ibcq);
4413	union rdma_cqe *cqe;
4414	u32 old_cons, new_cons;
4415	unsigned long flags;
4416	int update = 0;
4417	int done = 0;
4418
4419	if (cq->destroyed) {
4420	DP_ERR(dev,
4421	"warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4422	cq, cq->icid);
4423	return 0;
4424	}
4425
4426	if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4427	return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4428
4429	spin_lock_irqsave(&cq->cq_lock, flags);
4430	cqe = cq->latest_cqe;
4431	old_cons = qed_chain_get_cons_idx_u32(chain: &cq->pbl);
4432	while (num_entries && is_valid_cqe(cq, cqe)) {
4433	struct qedr_qp *qp;
4434	int cnt = 0;
4435
4436	/* prevent speculative reads of any field of CQE */
4437	rmb();
4438
4439	qp = cqe_get_qp(cqe);
4440	if (!qp) {
4441	WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4442	break;
4443	}
4444
4445	wc->qp = &qp->ibqp;
4446
4447	switch (cqe_get_type(cqe)) {
4448	case RDMA_CQE_TYPE_REQUESTER:
4449	cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4450	req: &cqe->req);
4451	try_consume_req_cqe(cq, qp, req: &cqe->req, update: &update);
4452	break;
4453	case RDMA_CQE_TYPE_RESPONDER_RQ:
4454	cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4455	resp: &cqe->resp, update: &update);
4456	break;
4457	case RDMA_CQE_TYPE_RESPONDER_SRQ:
4458	cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4459	wc, resp: &cqe->resp);
4460	update = 1;
4461	break;
4462	case RDMA_CQE_TYPE_INVALID:
4463	default:
4464	DP_ERR(dev, "Error: invalid CQE type = %d\n",
4465	cqe_get_type(cqe));
4466	}
4467	num_entries -= cnt;
4468	wc += cnt;
4469	done += cnt;
4470
4471	cqe = get_cqe(cq);
4472	}
4473	new_cons = qed_chain_get_cons_idx_u32(chain: &cq->pbl);
4474
4475	cq->cq_cons += new_cons - old_cons;
4476
4477	if (update)
4478	/* doorbell notifies abount latest VALID entry,
4479	* but chain already point to the next INVALID one
4480	*/
4481	doorbell_cq(cq, cons: cq->cq_cons - 1, flags: cq->arm_flags);
4482
4483	spin_unlock_irqrestore(lock: &cq->cq_lock, flags);
4484	return done;
4485	}
4486
4487	int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
4488	u32 port_num, const struct ib_wc *in_wc,
4489	const struct ib_grh *in_grh, const struct ib_mad *in,
4490	struct ib_mad *out_mad, size_t *out_mad_size,
4491	u16 *out_mad_pkey_index)
4492	{
4493	return IB_MAD_RESULT_SUCCESS;
4494	}
4495