1	/*
2	* Broadcom NetXtreme-E RoCE driver.
3	*
4	* Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5	* Broadcom refers to Broadcom Limited and/or its subsidiaries.
6	*
7	* This software is available to you under a choice of one of two
8	* licenses. You may choose to be licensed under the terms of the GNU
9	* General Public License (GPL) Version 2, available from the file
10	* COPYING in the main directory of this source tree, or the
11	* BSD license below:
12	*
13	* Redistribution and use in source and binary forms, with or without
14	* modification, are permitted provided that the following conditions
15	* are met:
16	*
17	* 1. Redistributions of source code must retain the above copyright
18	* notice, this list of conditions and the following disclaimer.
19	* 2. Redistributions in binary form must reproduce the above copyright
20	* notice, this list of conditions and the following disclaimer in
21	* the documentation and/or other materials provided with the
22	* distribution.
23	*
24	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25	* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26	* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27	* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28	* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29	* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30	* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31	* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32	* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33	* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34	* IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35	*
36	* Description: Main component of the bnxt_re driver
37	*/
38
39	#include <linux/module.h>
40	#include <linux/netdevice.h>
41	#include <linux/ethtool.h>
42	#include <linux/mutex.h>
43	#include <linux/list.h>
44	#include <linux/rculist.h>
45	#include <linux/spinlock.h>
46	#include <linux/pci.h>
47	#include <net/dcbnl.h>
48	#include <net/ipv6.h>
49	#include <net/addrconf.h>
50	#include <linux/if_ether.h>
51	#include <linux/auxiliary_bus.h>
52
53	#include <rdma/ib_verbs.h>
54	#include <rdma/ib_user_verbs.h>
55	#include <rdma/ib_umem.h>
56	#include <rdma/ib_addr.h>
57
58	#include "bnxt_ulp.h"
59	#include "roce_hsi.h"
60	#include "qplib_res.h"
61	#include "qplib_sp.h"
62	#include "qplib_fp.h"
63	#include "qplib_rcfw.h"
64	#include "bnxt_re.h"
65	#include "ib_verbs.h"
66	#include <rdma/bnxt_re-abi.h>
67	#include "bnxt.h"
68	#include "hw_counters.h"
69
70	static char version[] =
71	BNXT_RE_DESC "\n";
72
73	MODULE_AUTHOR("Eddie Wai <eddie.wai@broadcom.com>");
74	MODULE_DESCRIPTION(BNXT_RE_DESC " Driver");
75	MODULE_LICENSE("Dual BSD/GPL");
76
77	/* globals */
78	static DEFINE_MUTEX(bnxt_re_mutex);
79
80	static void bnxt_re_stop_irq(void *handle);
81	static void bnxt_re_dev_stop(struct bnxt_re_dev *rdev);
82	static int bnxt_re_netdev_event(struct notifier_block *notifier,
83	unsigned long event, void *ptr);
84	static struct bnxt_re_dev *bnxt_re_from_netdev(struct net_device *netdev);
85	static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev);
86	static int bnxt_re_hwrm_qcaps(struct bnxt_re_dev *rdev);
87
88	static int bnxt_re_hwrm_qcfg(struct bnxt_re_dev *rdev, u32 *db_len,
89	u32 *offset);
90	static void bnxt_re_set_db_offset(struct bnxt_re_dev *rdev)
91	{
92	struct bnxt_qplib_chip_ctx *cctx;
93	struct bnxt_en_dev *en_dev;
94	struct bnxt_qplib_res *res;
95	u32 l2db_len = 0;
96	u32 offset = 0;
97	u32 barlen;
98	int rc;
99
100	res = &rdev->qplib_res;
101	en_dev = rdev->en_dev;
102	cctx = rdev->chip_ctx;
103
104	/* Issue qcfg */
105	rc = bnxt_re_hwrm_qcfg(rdev, db_len: &l2db_len, offset: &offset);
106	if (rc)
107	dev_info(rdev_to_dev(rdev),
108	"Couldn't get DB bar size, Low latency framework is disabled\n");
109	/* set register offsets for both UC and WC */
110	res->dpi_tbl.ucreg.offset = res->is_vf ? BNXT_QPLIB_DBR_VF_DB_OFFSET :
111	BNXT_QPLIB_DBR_PF_DB_OFFSET;
112	res->dpi_tbl.wcreg.offset = res->dpi_tbl.ucreg.offset;
113
114	/* If WC mapping is disabled by L2 driver then en_dev->l2_db_size
115	* is equal to the DB-Bar actual size. This indicates that L2
116	* is mapping entire bar as UC-. RoCE driver can't enable WC mapping
117	* in such cases and DB-push will be disabled.
118	*/
119	barlen = pci_resource_len(res->pdev, RCFW_DBR_PCI_BAR_REGION);
120	if (cctx->modes.db_push && l2db_len && en_dev->l2_db_size != barlen) {
121	res->dpi_tbl.wcreg.offset = en_dev->l2_db_size;
122	dev_info(rdev_to_dev(rdev), "Low latency framework is enabled\n");
123	}
124	}
125
126	static void bnxt_re_set_drv_mode(struct bnxt_re_dev *rdev, u8 mode)
127	{
128	struct bnxt_qplib_chip_ctx *cctx;
129
130	cctx = rdev->chip_ctx;
131	cctx->modes.wqe_mode = bnxt_qplib_is_chip_gen_p5(cctx: rdev->chip_ctx) ?
132	mode : BNXT_QPLIB_WQE_MODE_STATIC;
133	if (bnxt_re_hwrm_qcaps(rdev))
134	dev_err(rdev_to_dev(rdev),
135	"Failed to query hwrm qcaps\n");
136	}
137
138	static void bnxt_re_destroy_chip_ctx(struct bnxt_re_dev *rdev)
139	{
140	struct bnxt_qplib_chip_ctx *chip_ctx;
141
142	if (!rdev->chip_ctx)
143	return;
144	chip_ctx = rdev->chip_ctx;
145	rdev->chip_ctx = NULL;
146	rdev->rcfw.res = NULL;
147	rdev->qplib_res.cctx = NULL;
148	rdev->qplib_res.pdev = NULL;
149	rdev->qplib_res.netdev = NULL;
150	kfree(objp: chip_ctx);
151	}
152
153	static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev, u8 wqe_mode)
154	{
155	struct bnxt_qplib_chip_ctx *chip_ctx;
156	struct bnxt_en_dev *en_dev;
157	int rc;
158
159	en_dev = rdev->en_dev;
160
161	chip_ctx = kzalloc(size: sizeof(*chip_ctx), GFP_KERNEL);
162	if (!chip_ctx)
163	return -ENOMEM;
164	chip_ctx->chip_num = en_dev->chip_num;
165	chip_ctx->hw_stats_size = en_dev->hw_ring_stats_size;
166
167	rdev->chip_ctx = chip_ctx;
168	/* rest members to follow eventually */
169
170	rdev->qplib_res.cctx = rdev->chip_ctx;
171	rdev->rcfw.res = &rdev->qplib_res;
172	rdev->qplib_res.dattr = &rdev->dev_attr;
173	rdev->qplib_res.is_vf = BNXT_EN_VF(en_dev);
174
175	bnxt_re_set_drv_mode(rdev, mode: wqe_mode);
176
177	bnxt_re_set_db_offset(rdev);
178	rc = bnxt_qplib_map_db_bar(res: &rdev->qplib_res);
179	if (rc)
180	return rc;
181
182	if (bnxt_qplib_determine_atomics(dev: en_dev->pdev))
183	ibdev_info(ibdev: &rdev->ibdev,
184	format: "platform doesn't support global atomics.");
185	return 0;
186	}
187
188	/* SR-IOV helper functions */
189
190	static void bnxt_re_get_sriov_func_type(struct bnxt_re_dev *rdev)
191	{
192	if (BNXT_EN_VF(rdev->en_dev))
193	rdev->is_virtfn = 1;
194	}
195
196	/* Set the maximum number of each resource that the driver actually wants
197	* to allocate. This may be up to the maximum number the firmware has
198	* reserved for the function. The driver may choose to allocate fewer
199	* resources than the firmware maximum.
200	*/
201	static void bnxt_re_limit_pf_res(struct bnxt_re_dev *rdev)
202	{
203	struct bnxt_qplib_dev_attr *attr;
204	struct bnxt_qplib_ctx *ctx;
205	int i;
206
207	attr = &rdev->dev_attr;
208	ctx = &rdev->qplib_ctx;
209
210	ctx->qpc_count = min_t(u32, BNXT_RE_MAX_QPC_COUNT,
211	attr->max_qp);
212	ctx->mrw_count = BNXT_RE_MAX_MRW_COUNT_256K;
213	/* Use max_mr from fw since max_mrw does not get set */
214	ctx->mrw_count = min_t(u32, ctx->mrw_count, attr->max_mr);
215	ctx->srqc_count = min_t(u32, BNXT_RE_MAX_SRQC_COUNT,
216	attr->max_srq);
217	ctx->cq_count = min_t(u32, BNXT_RE_MAX_CQ_COUNT, attr->max_cq);
218	if (!bnxt_qplib_is_chip_gen_p5(cctx: rdev->chip_ctx))
219	for (i = 0; i < MAX_TQM_ALLOC_REQ; i++)
220	rdev->qplib_ctx.tqm_ctx.qcount[i] =
221	rdev->dev_attr.tqm_alloc_reqs[i];
222	}
223
224	static void bnxt_re_limit_vf_res(struct bnxt_qplib_ctx *qplib_ctx, u32 num_vf)
225	{
226	struct bnxt_qplib_vf_res *vf_res;
227	u32 mrws = 0;
228	u32 vf_pct;
229	u32 nvfs;
230
231	vf_res = &qplib_ctx->vf_res;
232	/*
233	* Reserve a set of resources for the PF. Divide the remaining
234	* resources among the VFs
235	*/
236	vf_pct = 100 - BNXT_RE_PCT_RSVD_FOR_PF;
237	nvfs = num_vf;
238	num_vf = 100 * num_vf;
239	vf_res->max_qp_per_vf = (qplib_ctx->qpc_count * vf_pct) / num_vf;
240	vf_res->max_srq_per_vf = (qplib_ctx->srqc_count * vf_pct) / num_vf;
241	vf_res->max_cq_per_vf = (qplib_ctx->cq_count * vf_pct) / num_vf;
242	/*
243	* The driver allows many more MRs than other resources. If the
244	* firmware does also, then reserve a fixed amount for the PF and
245	* divide the rest among VFs. VFs may use many MRs for NFS
246	* mounts, ISER, NVME applications, etc. If the firmware severely
247	* restricts the number of MRs, then let PF have half and divide
248	* the rest among VFs, as for the other resource types.
249	*/
250	if (qplib_ctx->mrw_count < BNXT_RE_MAX_MRW_COUNT_64K) {
251	mrws = qplib_ctx->mrw_count * vf_pct;
252	nvfs = num_vf;
253	} else {
254	mrws = qplib_ctx->mrw_count - BNXT_RE_RESVD_MR_FOR_PF;
255	}
256	vf_res->max_mrw_per_vf = (mrws / nvfs);
257	vf_res->max_gid_per_vf = BNXT_RE_MAX_GID_PER_VF;
258	}
259
260	static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev)
261	{
262	u32 num_vfs;
263
264	memset(&rdev->qplib_ctx.vf_res, 0, sizeof(struct bnxt_qplib_vf_res));
265	bnxt_re_limit_pf_res(rdev);
266
267	num_vfs = bnxt_qplib_is_chip_gen_p5(cctx: rdev->chip_ctx) ?
268	BNXT_RE_GEN_P5_MAX_VF : rdev->num_vfs;
269	if (num_vfs)
270	bnxt_re_limit_vf_res(qplib_ctx: &rdev->qplib_ctx, num_vf: num_vfs);
271	}
272
273	static void bnxt_re_vf_res_config(struct bnxt_re_dev *rdev)
274	{
275
276	if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
277	return;
278	rdev->num_vfs = pci_sriov_get_totalvfs(dev: rdev->en_dev->pdev);
279	if (!bnxt_qplib_is_chip_gen_p5(cctx: rdev->chip_ctx)) {
280	bnxt_re_set_resource_limits(rdev);
281	bnxt_qplib_set_func_resources(res: &rdev->qplib_res, rcfw: &rdev->rcfw,
282	ctx: &rdev->qplib_ctx);
283	}
284	}
285
286	static void bnxt_re_shutdown(struct auxiliary_device *adev)
287	{
288	struct bnxt_re_dev *rdev = auxiliary_get_drvdata(auxdev: adev);
289
290	if (!rdev)
291	return;
292	ib_unregister_device(device: &rdev->ibdev);
293	bnxt_re_dev_uninit(rdev);
294	}
295
296	static void bnxt_re_stop_irq(void *handle)
297	{
298	struct bnxt_re_dev *rdev = (struct bnxt_re_dev *)handle;
299	struct bnxt_qplib_rcfw *rcfw = &rdev->rcfw;
300	struct bnxt_qplib_nq *nq;
301	int indx;
302
303	for (indx = BNXT_RE_NQ_IDX; indx < rdev->num_msix; indx++) {
304	nq = &rdev->nq[indx - 1];
305	bnxt_qplib_nq_stop_irq(nq, kill: false);
306	}
307
308	bnxt_qplib_rcfw_stop_irq(rcfw, kill: false);
309	}
310
311	static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent)
312	{
313	struct bnxt_re_dev *rdev = (struct bnxt_re_dev *)handle;
314	struct bnxt_msix_entry *msix_ent = rdev->en_dev->msix_entries;
315	struct bnxt_qplib_rcfw *rcfw = &rdev->rcfw;
316	struct bnxt_qplib_nq *nq;
317	int indx, rc;
318
319	if (!ent) {
320	/* Not setting the f/w timeout bit in rcfw.
321	* During the driver unload the first command
322	* to f/w will timeout and that will set the
323	* timeout bit.
324	*/
325	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to re-start IRQs\n");
326	return;
327	}
328
329	/* Vectors may change after restart, so update with new vectors
330	* in device sctructure.
331	*/
332	for (indx = 0; indx < rdev->num_msix; indx++)
333	rdev->en_dev->msix_entries[indx].vector = ent[indx].vector;
334
335	rc = bnxt_qplib_rcfw_start_irq(rcfw, msix_ent[BNXT_RE_AEQ_IDX].vector,
336	false);
337	if (rc) {
338	ibdev_warn(ibdev: &rdev->ibdev, format: "Failed to reinit CREQ\n");
339	return;
340	}
341	for (indx = BNXT_RE_NQ_IDX ; indx < rdev->num_msix; indx++) {
342	nq = &rdev->nq[indx - 1];
343	rc = bnxt_qplib_nq_start_irq(nq, indx - 1,
344	msix_ent[indx].vector, false);
345	if (rc) {
346	ibdev_warn(ibdev: &rdev->ibdev, format: "Failed to reinit NQ index %d\n",
347	indx - 1);
348	return;
349	}
350	}
351	}
352
353	static struct bnxt_ulp_ops bnxt_re_ulp_ops = {
354	.ulp_irq_stop = bnxt_re_stop_irq,
355	.ulp_irq_restart = bnxt_re_start_irq
356	};
357
358	/* RoCE -> Net driver */
359
360	static int bnxt_re_register_netdev(struct bnxt_re_dev *rdev)
361	{
362	struct bnxt_en_dev *en_dev;
363	int rc;
364
365	en_dev = rdev->en_dev;
366
367	rc = bnxt_register_dev(en_dev, &bnxt_re_ulp_ops, rdev);
368	if (!rc)
369	rdev->qplib_res.pdev = rdev->en_dev->pdev;
370	return rc;
371	}
372
373	static void bnxt_re_init_hwrm_hdr(struct input *hdr, u16 opcd)
374	{
375	hdr->req_type = cpu_to_le16(opcd);
376	hdr->cmpl_ring = cpu_to_le16(-1);
377	hdr->target_id = cpu_to_le16(-1);
378	}
379
380	static void bnxt_re_fill_fw_msg(struct bnxt_fw_msg *fw_msg, void *msg,
381	int msg_len, void *resp, int resp_max_len,
382	int timeout)
383	{
384	fw_msg->msg = msg;
385	fw_msg->msg_len = msg_len;
386	fw_msg->resp = resp;
387	fw_msg->resp_max_len = resp_max_len;
388	fw_msg->timeout = timeout;
389	}
390
391	/* Query device config using common hwrm */
392	static int bnxt_re_hwrm_qcfg(struct bnxt_re_dev *rdev, u32 *db_len,
393	u32 *offset)
394	{
395	struct bnxt_en_dev *en_dev = rdev->en_dev;
396	struct hwrm_func_qcfg_output resp = {0};
397	struct hwrm_func_qcfg_input req = {0};
398	struct bnxt_fw_msg fw_msg = {};
399	int rc;
400
401	bnxt_re_init_hwrm_hdr(hdr: (void *)&req, opcd: HWRM_FUNC_QCFG);
402	req.fid = cpu_to_le16(0xffff);
403	bnxt_re_fill_fw_msg(fw_msg: &fw_msg, msg: (void *)&req, msg_len: sizeof(req), resp: (void *)&resp,
404	resp_max_len: sizeof(resp), timeout: DFLT_HWRM_CMD_TIMEOUT);
405	rc = bnxt_send_msg(en_dev, &fw_msg);
406	if (!rc) {
407	*db_len = PAGE_ALIGN(le16_to_cpu(resp.l2_doorbell_bar_size_kb) * 1024);
408	*offset = PAGE_ALIGN(le16_to_cpu(resp.legacy_l2_db_size_kb) * 1024);
409	}
410	return rc;
411	}
412
413	/* Query function capabilities using common hwrm */
414	int bnxt_re_hwrm_qcaps(struct bnxt_re_dev *rdev)
415	{
416	struct bnxt_en_dev *en_dev = rdev->en_dev;
417	struct hwrm_func_qcaps_output resp = {};
418	struct hwrm_func_qcaps_input req = {};
419	struct bnxt_qplib_chip_ctx *cctx;
420	struct bnxt_fw_msg fw_msg = {};
421	int rc;
422
423	cctx = rdev->chip_ctx;
424	bnxt_re_init_hwrm_hdr(hdr: (void *)&req, opcd: HWRM_FUNC_QCAPS);
425	req.fid = cpu_to_le16(0xffff);
426	bnxt_re_fill_fw_msg(fw_msg: &fw_msg, msg: (void *)&req, msg_len: sizeof(req), resp: (void *)&resp,
427	resp_max_len: sizeof(resp), timeout: DFLT_HWRM_CMD_TIMEOUT);
428
429	rc = bnxt_send_msg(en_dev, &fw_msg);
430	if (rc)
431	return rc;
432	cctx->modes.db_push = le32_to_cpu(resp.flags) & FUNC_QCAPS_RESP_FLAGS_WCB_PUSH_MODE;
433
434	cctx->modes.dbr_pacing =
435	le32_to_cpu(resp.flags_ext2) &
436	FUNC_QCAPS_RESP_FLAGS_EXT2_DBR_PACING_EXT_SUPPORTED;
437	return 0;
438	}
439
440	static int bnxt_re_hwrm_dbr_pacing_qcfg(struct bnxt_re_dev *rdev)
441	{
442	struct hwrm_func_dbr_pacing_qcfg_output resp = {};
443	struct hwrm_func_dbr_pacing_qcfg_input req = {};
444	struct bnxt_en_dev *en_dev = rdev->en_dev;
445	struct bnxt_qplib_chip_ctx *cctx;
446	struct bnxt_fw_msg fw_msg = {};
447	int rc;
448
449	cctx = rdev->chip_ctx;
450	bnxt_re_init_hwrm_hdr(hdr: (void *)&req, opcd: HWRM_FUNC_DBR_PACING_QCFG);
451	bnxt_re_fill_fw_msg(fw_msg: &fw_msg, msg: (void *)&req, msg_len: sizeof(req), resp: (void *)&resp,
452	resp_max_len: sizeof(resp), timeout: DFLT_HWRM_CMD_TIMEOUT);
453	rc = bnxt_send_msg(en_dev, &fw_msg);
454	if (rc)
455	return rc;
456
457	if ((le32_to_cpu(resp.dbr_stat_db_fifo_reg) &
458	FUNC_DBR_PACING_QCFG_RESP_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK) ==
459	FUNC_DBR_PACING_QCFG_RESP_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_GRC)
460	cctx->dbr_stat_db_fifo =
461	le32_to_cpu(resp.dbr_stat_db_fifo_reg) &
462	~FUNC_DBR_PACING_QCFG_RESP_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK;
463	return 0;
464	}
465
466	/* Update the pacing tunable parameters to the default values */
467	static void bnxt_re_set_default_pacing_data(struct bnxt_re_dev *rdev)
468	{
469	struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data;
470
471	pacing_data->do_pacing = rdev->pacing.dbr_def_do_pacing;
472	pacing_data->pacing_th = rdev->pacing.pacing_algo_th;
473	pacing_data->alarm_th =
474	pacing_data->pacing_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE;
475	}
476
477	static void __wait_for_fifo_occupancy_below_th(struct bnxt_re_dev *rdev)
478	{
479	u32 read_val, fifo_occup;
480
481	/* loop shouldn't run infintely as the occupancy usually goes
482	* below pacing algo threshold as soon as pacing kicks in.
483	*/
484	while (1) {
485	read_val = readl(addr: rdev->en_dev->bar0 + rdev->pacing.dbr_db_fifo_reg_off);
486	fifo_occup = BNXT_RE_MAX_FIFO_DEPTH -
487	((read_val & BNXT_RE_DB_FIFO_ROOM_MASK) >>
488	BNXT_RE_DB_FIFO_ROOM_SHIFT);
489	/* Fifo occupancy cannot be greater the MAX FIFO depth */
490	if (fifo_occup > BNXT_RE_MAX_FIFO_DEPTH)
491	break;
492
493	if (fifo_occup < rdev->qplib_res.pacing_data->pacing_th)
494	break;
495	}
496	}
497
498	static void bnxt_re_db_fifo_check(struct work_struct *work)
499	{
500	struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev,
501	dbq_fifo_check_work);
502	struct bnxt_qplib_db_pacing_data *pacing_data;
503	u32 pacing_save;
504
505	if (!mutex_trylock(lock: &rdev->pacing.dbq_lock))
506	return;
507	pacing_data = rdev->qplib_res.pacing_data;
508	pacing_save = rdev->pacing.do_pacing_save;
509	__wait_for_fifo_occupancy_below_th(rdev);
510	cancel_delayed_work_sync(dwork: &rdev->dbq_pacing_work);
511	if (pacing_save > rdev->pacing.dbr_def_do_pacing) {
512	/* Double the do_pacing value during the congestion */
513	pacing_save = pacing_save << 1;
514	} else {
515	/*
516	* when a new congestion is detected increase the do_pacing
517	* by 8 times. And also increase the pacing_th by 4 times. The
518	* reason to increase pacing_th is to give more space for the
519	* queue to oscillate down without getting empty, but also more
520	* room for the queue to increase without causing another alarm.
521	*/
522	pacing_save = pacing_save << 3;
523	pacing_data->pacing_th = rdev->pacing.pacing_algo_th * 4;
524	}
525
526	if (pacing_save > BNXT_RE_MAX_DBR_DO_PACING)
527	pacing_save = BNXT_RE_MAX_DBR_DO_PACING;
528
529	pacing_data->do_pacing = pacing_save;
530	rdev->pacing.do_pacing_save = pacing_data->do_pacing;
531	pacing_data->alarm_th =
532	pacing_data->pacing_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE;
533	schedule_delayed_work(dwork: &rdev->dbq_pacing_work,
534	delay: msecs_to_jiffies(m: rdev->pacing.dbq_pacing_time));
535	rdev->stats.pacing.alerts++;
536	mutex_unlock(lock: &rdev->pacing.dbq_lock);
537	}
538
539	static void bnxt_re_pacing_timer_exp(struct work_struct *work)
540	{
541	struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev,
542	dbq_pacing_work.work);
543	struct bnxt_qplib_db_pacing_data *pacing_data;
544	u32 read_val, fifo_occup;
545
546	if (!mutex_trylock(lock: &rdev->pacing.dbq_lock))
547	return;
548
549	pacing_data = rdev->qplib_res.pacing_data;
550	read_val = readl(addr: rdev->en_dev->bar0 + rdev->pacing.dbr_db_fifo_reg_off);
551	fifo_occup = BNXT_RE_MAX_FIFO_DEPTH -
552	((read_val & BNXT_RE_DB_FIFO_ROOM_MASK) >>
553	BNXT_RE_DB_FIFO_ROOM_SHIFT);
554
555	if (fifo_occup > pacing_data->pacing_th)
556	goto restart_timer;
557
558	/*
559	* Instead of immediately going back to the default do_pacing
560	* reduce it by 1/8 times and restart the timer.
561	*/
562	pacing_data->do_pacing = pacing_data->do_pacing - (pacing_data->do_pacing >> 3);
563	pacing_data->do_pacing = max_t(u32, rdev->pacing.dbr_def_do_pacing, pacing_data->do_pacing);
564	if (pacing_data->do_pacing <= rdev->pacing.dbr_def_do_pacing) {
565	bnxt_re_set_default_pacing_data(rdev);
566	rdev->stats.pacing.complete++;
567	goto dbq_unlock;
568	}
569
570	restart_timer:
571	schedule_delayed_work(dwork: &rdev->dbq_pacing_work,
572	delay: msecs_to_jiffies(m: rdev->pacing.dbq_pacing_time));
573	rdev->stats.pacing.resched++;
574	dbq_unlock:
575	rdev->pacing.do_pacing_save = pacing_data->do_pacing;
576	mutex_unlock(lock: &rdev->pacing.dbq_lock);
577	}
578
579	void bnxt_re_pacing_alert(struct bnxt_re_dev *rdev)
580	{
581	struct bnxt_qplib_db_pacing_data *pacing_data;
582
583	if (!rdev->pacing.dbr_pacing)
584	return;
585	mutex_lock(&rdev->pacing.dbq_lock);
586	pacing_data = rdev->qplib_res.pacing_data;
587
588	/*
589	* Increase the alarm_th to max so that other user lib instances do not
590	* keep alerting the driver.
591	*/
592	pacing_data->alarm_th = BNXT_RE_MAX_FIFO_DEPTH;
593	pacing_data->do_pacing = BNXT_RE_MAX_DBR_DO_PACING;
594	cancel_work_sync(work: &rdev->dbq_fifo_check_work);
595	schedule_work(work: &rdev->dbq_fifo_check_work);
596	mutex_unlock(lock: &rdev->pacing.dbq_lock);
597	}
598
599	static int bnxt_re_initialize_dbr_pacing(struct bnxt_re_dev *rdev)
600	{
601	if (bnxt_re_hwrm_dbr_pacing_qcfg(rdev))
602	return -EIO;
603
604	/* Allocate a page for app use */
605	rdev->pacing.dbr_page = (void *)__get_free_page(GFP_KERNEL);
606	if (!rdev->pacing.dbr_page)
607	return -ENOMEM;
608
609	memset((u8 *)rdev->pacing.dbr_page, 0, PAGE_SIZE);
610	rdev->qplib_res.pacing_data = (struct bnxt_qplib_db_pacing_data *)rdev->pacing.dbr_page;
611
612	/* MAP HW window 2 for reading db fifo depth */
613	writel(val: rdev->chip_ctx->dbr_stat_db_fifo & BNXT_GRC_BASE_MASK,
614	addr: rdev->en_dev->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
615	rdev->pacing.dbr_db_fifo_reg_off =
616	(rdev->chip_ctx->dbr_stat_db_fifo & BNXT_GRC_OFFSET_MASK) +
617	BNXT_RE_GRC_FIFO_REG_BASE;
618	rdev->pacing.dbr_bar_addr =
619	pci_resource_start(rdev->qplib_res.pdev, 0) + rdev->pacing.dbr_db_fifo_reg_off;
620
621	rdev->pacing.pacing_algo_th = BNXT_RE_PACING_ALGO_THRESHOLD;
622	rdev->pacing.dbq_pacing_time = BNXT_RE_DBR_PACING_TIME;
623	rdev->pacing.dbr_def_do_pacing = BNXT_RE_DBR_DO_PACING_NO_CONGESTION;
624	rdev->pacing.do_pacing_save = rdev->pacing.dbr_def_do_pacing;
625	rdev->qplib_res.pacing_data->fifo_max_depth = BNXT_RE_MAX_FIFO_DEPTH;
626	rdev->qplib_res.pacing_data->fifo_room_mask = BNXT_RE_DB_FIFO_ROOM_MASK;
627	rdev->qplib_res.pacing_data->fifo_room_shift = BNXT_RE_DB_FIFO_ROOM_SHIFT;
628	rdev->qplib_res.pacing_data->grc_reg_offset = rdev->pacing.dbr_db_fifo_reg_off;
629	bnxt_re_set_default_pacing_data(rdev);
630	/* Initialize worker for DBR Pacing */
631	INIT_WORK(&rdev->dbq_fifo_check_work, bnxt_re_db_fifo_check);
632	INIT_DELAYED_WORK(&rdev->dbq_pacing_work, bnxt_re_pacing_timer_exp);
633	return 0;
634	}
635
636	static void bnxt_re_deinitialize_dbr_pacing(struct bnxt_re_dev *rdev)
637	{
638	cancel_work_sync(work: &rdev->dbq_fifo_check_work);
639	cancel_delayed_work_sync(dwork: &rdev->dbq_pacing_work);
640	if (rdev->pacing.dbr_page)
641	free_page((u64)rdev->pacing.dbr_page);
642
643	rdev->pacing.dbr_page = NULL;
644	rdev->pacing.dbr_pacing = false;
645	}
646
647	static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev,
648	u16 fw_ring_id, int type)
649	{
650	struct bnxt_en_dev *en_dev;
651	struct hwrm_ring_free_input req = {};
652	struct hwrm_ring_free_output resp;
653	struct bnxt_fw_msg fw_msg = {};
654	int rc = -EINVAL;
655
656	if (!rdev)
657	return rc;
658
659	en_dev = rdev->en_dev;
660
661	if (!en_dev)
662	return rc;
663
664	if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
665	return 0;
666
667	bnxt_re_init_hwrm_hdr(hdr: (void *)&req, opcd: HWRM_RING_FREE);
668	req.ring_type = type;
669	req.ring_id = cpu_to_le16(fw_ring_id);
670	bnxt_re_fill_fw_msg(fw_msg: &fw_msg, msg: (void *)&req, msg_len: sizeof(req), resp: (void *)&resp,
671	resp_max_len: sizeof(resp), timeout: DFLT_HWRM_CMD_TIMEOUT);
672	rc = bnxt_send_msg(en_dev, &fw_msg);
673	if (rc)
674	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to free HW ring:%d :%#x",
675	req.ring_id, rc);
676	return rc;
677	}
678
679	static int bnxt_re_net_ring_alloc(struct bnxt_re_dev *rdev,
680	struct bnxt_re_ring_attr *ring_attr,
681	u16 *fw_ring_id)
682	{
683	struct bnxt_en_dev *en_dev = rdev->en_dev;
684	struct hwrm_ring_alloc_input req = {};
685	struct hwrm_ring_alloc_output resp;
686	struct bnxt_fw_msg fw_msg = {};
687	int rc = -EINVAL;
688
689	if (!en_dev)
690	return rc;
691
692	bnxt_re_init_hwrm_hdr(hdr: (void *)&req, opcd: HWRM_RING_ALLOC);
693	req.enables = 0;
694	req.page_tbl_addr = cpu_to_le64(ring_attr->dma_arr[0]);
695	if (ring_attr->pages > 1) {
696	/* Page size is in log2 units */
697	req.page_size = BNXT_PAGE_SHIFT;
698	req.page_tbl_depth = 1;
699	}
700	req.fbo = 0;
701	/* Association of ring index with doorbell index and MSIX number */
702	req.logical_id = cpu_to_le16(ring_attr->lrid);
703	req.length = cpu_to_le32(ring_attr->depth + 1);
704	req.ring_type = ring_attr->type;
705	req.int_mode = ring_attr->mode;
706	bnxt_re_fill_fw_msg(fw_msg: &fw_msg, msg: (void *)&req, msg_len: sizeof(req), resp: (void *)&resp,
707	resp_max_len: sizeof(resp), timeout: DFLT_HWRM_CMD_TIMEOUT);
708	rc = bnxt_send_msg(en_dev, &fw_msg);
709	if (!rc)
710	*fw_ring_id = le16_to_cpu(resp.ring_id);
711
712	return rc;
713	}
714
715	static int bnxt_re_net_stats_ctx_free(struct bnxt_re_dev *rdev,
716	u32 fw_stats_ctx_id)
717	{
718	struct bnxt_en_dev *en_dev = rdev->en_dev;
719	struct hwrm_stat_ctx_free_input req = {};
720	struct hwrm_stat_ctx_free_output resp = {};
721	struct bnxt_fw_msg fw_msg = {};
722	int rc = -EINVAL;
723
724	if (!en_dev)
725	return rc;
726
727	if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
728	return 0;
729
730	bnxt_re_init_hwrm_hdr(hdr: (void *)&req, opcd: HWRM_STAT_CTX_FREE);
731	req.stat_ctx_id = cpu_to_le32(fw_stats_ctx_id);
732	bnxt_re_fill_fw_msg(fw_msg: &fw_msg, msg: (void *)&req, msg_len: sizeof(req), resp: (void *)&resp,
733	resp_max_len: sizeof(resp), timeout: DFLT_HWRM_CMD_TIMEOUT);
734	rc = bnxt_send_msg(en_dev, &fw_msg);
735	if (rc)
736	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to free HW stats context %#x",
737	rc);
738
739	return rc;
740	}
741
742	static int bnxt_re_net_stats_ctx_alloc(struct bnxt_re_dev *rdev,
743	dma_addr_t dma_map,
744	u32 *fw_stats_ctx_id)
745	{
746	struct bnxt_qplib_chip_ctx *chip_ctx = rdev->chip_ctx;
747	struct hwrm_stat_ctx_alloc_output resp = {};
748	struct hwrm_stat_ctx_alloc_input req = {};
749	struct bnxt_en_dev *en_dev = rdev->en_dev;
750	struct bnxt_fw_msg fw_msg = {};
751	int rc = -EINVAL;
752
753	*fw_stats_ctx_id = INVALID_STATS_CTX_ID;
754
755	if (!en_dev)
756	return rc;
757
758	bnxt_re_init_hwrm_hdr(hdr: (void *)&req, opcd: HWRM_STAT_CTX_ALLOC);
759	req.update_period_ms = cpu_to_le32(1000);
760	req.stats_dma_addr = cpu_to_le64(dma_map);
761	req.stats_dma_length = cpu_to_le16(chip_ctx->hw_stats_size);
762	req.stat_ctx_flags = STAT_CTX_ALLOC_REQ_STAT_CTX_FLAGS_ROCE;
763	bnxt_re_fill_fw_msg(fw_msg: &fw_msg, msg: (void *)&req, msg_len: sizeof(req), resp: (void *)&resp,
764	resp_max_len: sizeof(resp), timeout: DFLT_HWRM_CMD_TIMEOUT);
765	rc = bnxt_send_msg(en_dev, &fw_msg);
766	if (!rc)
767	*fw_stats_ctx_id = le32_to_cpu(resp.stat_ctx_id);
768
769	return rc;
770	}
771
772	static void bnxt_re_disassociate_ucontext(struct ib_ucontext *ibcontext)
773	{
774	}
775
776	/* Device */
777
778	static struct bnxt_re_dev *bnxt_re_from_netdev(struct net_device *netdev)
779	{
780	struct ib_device *ibdev =
781	ib_device_get_by_netdev(ndev: netdev, driver_id: RDMA_DRIVER_BNXT_RE);
782	if (!ibdev)
783	return NULL;
784
785	return container_of(ibdev, struct bnxt_re_dev, ibdev);
786	}
787
788	static ssize_t hw_rev_show(struct device *device, struct device_attribute *attr,
789	char *buf)
790	{
791	struct bnxt_re_dev *rdev =
792	rdma_device_to_drv_device(device, struct bnxt_re_dev, ibdev);
793
794	return sysfs_emit(buf, fmt: "0x%x\n", rdev->en_dev->pdev->vendor);
795	}
796	static DEVICE_ATTR_RO(hw_rev);
797
798	static ssize_t hca_type_show(struct device *device,
799	struct device_attribute *attr, char *buf)
800	{
801	struct bnxt_re_dev *rdev =
802	rdma_device_to_drv_device(device, struct bnxt_re_dev, ibdev);
803
804	return sysfs_emit(buf, fmt: "%s\n", rdev->ibdev.node_desc);
805	}
806	static DEVICE_ATTR_RO(hca_type);
807
808	static struct attribute *bnxt_re_attributes[] = {
809	&dev_attr_hw_rev.attr,
810	&dev_attr_hca_type.attr,
811	NULL
812	};
813
814	static const struct attribute_group bnxt_re_dev_attr_group = {
815	.attrs = bnxt_re_attributes,
816	};
817
818	static const struct ib_device_ops bnxt_re_dev_ops = {
819	.owner = THIS_MODULE,
820	.driver_id = RDMA_DRIVER_BNXT_RE,
821	.uverbs_abi_ver = BNXT_RE_ABI_VERSION,
822
823	.add_gid = bnxt_re_add_gid,
824	.alloc_hw_port_stats = bnxt_re_ib_alloc_hw_port_stats,
825	.alloc_mr = bnxt_re_alloc_mr,
826	.alloc_pd = bnxt_re_alloc_pd,
827	.alloc_ucontext = bnxt_re_alloc_ucontext,
828	.create_ah = bnxt_re_create_ah,
829	.create_cq = bnxt_re_create_cq,
830	.create_qp = bnxt_re_create_qp,
831	.create_srq = bnxt_re_create_srq,
832	.create_user_ah = bnxt_re_create_ah,
833	.dealloc_pd = bnxt_re_dealloc_pd,
834	.dealloc_ucontext = bnxt_re_dealloc_ucontext,
835	.del_gid = bnxt_re_del_gid,
836	.dereg_mr = bnxt_re_dereg_mr,
837	.destroy_ah = bnxt_re_destroy_ah,
838	.destroy_cq = bnxt_re_destroy_cq,
839	.destroy_qp = bnxt_re_destroy_qp,
840	.destroy_srq = bnxt_re_destroy_srq,
841	.device_group = &bnxt_re_dev_attr_group,
842	.disassociate_ucontext = bnxt_re_disassociate_ucontext,
843	.get_dev_fw_str = bnxt_re_query_fw_str,
844	.get_dma_mr = bnxt_re_get_dma_mr,
845	.get_hw_stats = bnxt_re_ib_get_hw_stats,
846	.get_link_layer = bnxt_re_get_link_layer,
847	.get_port_immutable = bnxt_re_get_port_immutable,
848	.map_mr_sg = bnxt_re_map_mr_sg,
849	.mmap = bnxt_re_mmap,
850	.mmap_free = bnxt_re_mmap_free,
851	.modify_qp = bnxt_re_modify_qp,
852	.modify_srq = bnxt_re_modify_srq,
853	.poll_cq = bnxt_re_poll_cq,
854	.post_recv = bnxt_re_post_recv,
855	.post_send = bnxt_re_post_send,
856	.post_srq_recv = bnxt_re_post_srq_recv,
857	.query_ah = bnxt_re_query_ah,
858	.query_device = bnxt_re_query_device,
859	.query_pkey = bnxt_re_query_pkey,
860	.query_port = bnxt_re_query_port,
861	.query_qp = bnxt_re_query_qp,
862	.query_srq = bnxt_re_query_srq,
863	.reg_user_mr = bnxt_re_reg_user_mr,
864	.reg_user_mr_dmabuf = bnxt_re_reg_user_mr_dmabuf,
865	.req_notify_cq = bnxt_re_req_notify_cq,
866	.resize_cq = bnxt_re_resize_cq,
867	INIT_RDMA_OBJ_SIZE(ib_ah, bnxt_re_ah, ib_ah),
868	INIT_RDMA_OBJ_SIZE(ib_cq, bnxt_re_cq, ib_cq),
869	INIT_RDMA_OBJ_SIZE(ib_pd, bnxt_re_pd, ib_pd),
870	INIT_RDMA_OBJ_SIZE(ib_qp, bnxt_re_qp, ib_qp),
871	INIT_RDMA_OBJ_SIZE(ib_srq, bnxt_re_srq, ib_srq),
872	INIT_RDMA_OBJ_SIZE(ib_ucontext, bnxt_re_ucontext, ib_uctx),
873	};
874
875	static int bnxt_re_register_ib(struct bnxt_re_dev *rdev)
876	{
877	struct ib_device *ibdev = &rdev->ibdev;
878	int ret;
879
880	/* ib device init */
881	ibdev->node_type = RDMA_NODE_IB_CA;
882	strscpy(p: ibdev->node_desc, BNXT_RE_DESC " HCA",
883	strlen(BNXT_RE_DESC) + 5);
884	ibdev->phys_port_cnt = 1;
885
886	addrconf_addr_eui48(eui: (u8 *)&ibdev->node_guid, addr: rdev->netdev->dev_addr);
887
888	ibdev->num_comp_vectors = rdev->num_msix - 1;
889	ibdev->dev.parent = &rdev->en_dev->pdev->dev;
890	ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY;
891
892	if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS))
893	ibdev->driver_def = bnxt_re_uapi_defs;
894
895	ib_set_device_ops(device: ibdev, ops: &bnxt_re_dev_ops);
896	ret = ib_device_set_netdev(ib_dev: &rdev->ibdev, ndev: rdev->netdev, port: 1);
897	if (ret)
898	return ret;
899
900	dma_set_max_seg_size(dev: &rdev->en_dev->pdev->dev, UINT_MAX);
901	ibdev->uverbs_cmd_mask |= BIT_ULL(IB_USER_VERBS_CMD_POLL_CQ);
902	return ib_register_device(device: ibdev, name: "bnxt_re%d", dma_device: &rdev->en_dev->pdev->dev);
903	}
904
905	static struct bnxt_re_dev *bnxt_re_dev_add(struct bnxt_aux_priv *aux_priv,
906	struct bnxt_en_dev *en_dev)
907	{
908	struct bnxt_re_dev *rdev;
909
910	/* Allocate bnxt_re_dev instance here */
911	rdev = ib_alloc_device(bnxt_re_dev, ibdev);
912	if (!rdev) {
913	ibdev_err(NULL, format: "%s: bnxt_re_dev allocation failure!",
914	ROCE_DRV_MODULE_NAME);
915	return NULL;
916	}
917	/* Default values */
918	rdev->nb.notifier_call = NULL;
919	rdev->netdev = en_dev->net;
920	rdev->en_dev = en_dev;
921	rdev->id = rdev->en_dev->pdev->devfn;
922	INIT_LIST_HEAD(list: &rdev->qp_list);
923	mutex_init(&rdev->qp_lock);
924	mutex_init(&rdev->pacing.dbq_lock);
925	atomic_set(v: &rdev->stats.res.qp_count, i: 0);
926	atomic_set(v: &rdev->stats.res.cq_count, i: 0);
927	atomic_set(v: &rdev->stats.res.srq_count, i: 0);
928	atomic_set(v: &rdev->stats.res.mr_count, i: 0);
929	atomic_set(v: &rdev->stats.res.mw_count, i: 0);
930	atomic_set(v: &rdev->stats.res.ah_count, i: 0);
931	atomic_set(v: &rdev->stats.res.pd_count, i: 0);
932	rdev->cosq[0] = 0xFFFF;
933	rdev->cosq[1] = 0xFFFF;
934
935	return rdev;
936	}
937
938	static int bnxt_re_handle_unaffi_async_event(struct creq_func_event
939	*unaffi_async)
940	{
941	switch (unaffi_async->event) {
942	case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR:
943	break;
944	case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR:
945	break;
946	case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR:
947	break;
948	case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR:
949	break;
950	case CREQ_FUNC_EVENT_EVENT_CQ_ERROR:
951	break;
952	case CREQ_FUNC_EVENT_EVENT_TQM_ERROR:
953	break;
954	case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR:
955	break;
956	case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR:
957	break;
958	case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR:
959	break;
960	case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR:
961	break;
962	case CREQ_FUNC_EVENT_EVENT_TIM_ERROR:
963	break;
964	default:
965	return -EINVAL;
966	}
967	return 0;
968	}
969
970	static int bnxt_re_handle_qp_async_event(struct creq_qp_event *qp_event,
971	struct bnxt_re_qp *qp)
972	{
973	struct bnxt_re_srq *srq = container_of(qp->qplib_qp.srq, struct bnxt_re_srq,
974	qplib_srq);
975	struct creq_qp_error_notification *err_event;
976	struct ib_event event = {};
977	unsigned int flags;
978
979	if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR &&
980	rdma_is_kernel_res(res: &qp->ib_qp.res)) {
981	flags = bnxt_re_lock_cqs(qp);
982	bnxt_qplib_add_flush_qp(qp: &qp->qplib_qp);
983	bnxt_re_unlock_cqs(qp, flags);
984	}
985
986	event.device = &qp->rdev->ibdev;
987	event.element.qp = &qp->ib_qp;
988	event.event = IB_EVENT_QP_FATAL;
989
990	err_event = (struct creq_qp_error_notification *)qp_event;
991
992	switch (err_event->req_err_state_reason) {
993	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_OPCODE_ERROR:
994	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_TIMEOUT_RETRY_LIMIT:
995	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_RNR_TIMEOUT_RETRY_LIMIT:
996	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_NAK_ARRIVAL_2:
997	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_NAK_ARRIVAL_3:
998	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_INVALID_READ_RESP:
999	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_ILLEGAL_BIND:
1000	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_ILLEGAL_FAST_REG:
1001	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_ILLEGAL_INVALIDATE:
1002	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_RETRAN_LOCAL_ERROR:
1003	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_AV_DOMAIN_ERROR:
1004	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_PROD_WQE_MSMTCH_ERROR:
1005	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_PSN_RANGE_CHECK_ERROR:
1006	event.event = IB_EVENT_QP_ACCESS_ERR;
1007	break;
1008	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_NAK_ARRIVAL_1:
1009	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_NAK_ARRIVAL_4:
1010	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_READ_RESP_LENGTH:
1011	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_WQE_FORMAT_ERROR:
1012	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_ORRQ_FORMAT_ERROR:
1013	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_INVALID_AVID_ERROR:
1014	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_SERV_TYPE_ERROR:
1015	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_INVALID_OP_ERROR:
1016	event.event = IB_EVENT_QP_REQ_ERR;
1017	break;
1018	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_RX_MEMORY_ERROR:
1019	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_TX_MEMORY_ERROR:
1020	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_CMP_ERROR:
1021	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_CQ_LOAD_ERROR:
1022	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_TX_PCI_ERROR:
1023	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_RX_PCI_ERROR:
1024	case CREQ_QP_ERROR_NOTIFICATION_REQ_ERR_STATE_REASON_REQ_RETX_SETUP_ERROR:
1025	event.event = IB_EVENT_QP_FATAL;
1026	break;
1027
1028	default:
1029	break;
1030	}
1031
1032	switch (err_event->res_err_state_reason) {
1033	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_EXCEED_MAX:
1034	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_PAYLOAD_LENGTH_MISMATCH:
1035	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_PSN_SEQ_ERROR_RETRY_LIMIT:
1036	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_RX_INVALID_R_KEY:
1037	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_RX_DOMAIN_ERROR:
1038	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_RX_NO_PERMISSION:
1039	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_RX_RANGE_ERROR:
1040	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_TX_INVALID_R_KEY:
1041	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_TX_DOMAIN_ERROR:
1042	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_TX_NO_PERMISSION:
1043	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_TX_RANGE_ERROR:
1044	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_UNALIGN_ATOMIC:
1045	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_PSN_NOT_FOUND:
1046	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_INVALID_DUP_RKEY:
1047	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_IRRQ_FORMAT_ERROR:
1048	event.event = IB_EVENT_QP_ACCESS_ERR;
1049	break;
1050	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_EXCEEDS_WQE:
1051	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_WQE_FORMAT_ERROR:
1052	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_UNSUPPORTED_OPCODE:
1053	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_REM_INVALIDATE:
1054	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_OPCODE_ERROR:
1055	event.event = IB_EVENT_QP_REQ_ERR;
1056	break;
1057	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_IRRQ_OFLOW:
1058	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_CMP_ERROR:
1059	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_CQ_LOAD_ERROR:
1060	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_TX_PCI_ERROR:
1061	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_RX_PCI_ERROR:
1062	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_MEMORY_ERROR:
1063	event.event = IB_EVENT_QP_FATAL;
1064	break;
1065	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_SRQ_LOAD_ERROR:
1066	case CREQ_QP_ERROR_NOTIFICATION_RES_ERR_STATE_REASON_RES_SRQ_ERROR:
1067	if (srq)
1068	event.event = IB_EVENT_SRQ_ERR;
1069	break;
1070	default:
1071	break;
1072	}
1073
1074	if (err_event->res_err_state_reason || err_event->req_err_state_reason) {
1075	ibdev_dbg(&qp->rdev->ibdev,
1076	"%s %s qp_id: %d cons (%d %d) req (%d %d) res (%d %d)\n",
1077	__func__, rdma_is_kernel_res(&qp->ib_qp.res) ? "kernel" : "user",
1078	qp->qplib_qp.id,
1079	err_event->sq_cons_idx,
1080	err_event->rq_cons_idx,
1081	err_event->req_slow_path_state,
1082	err_event->req_err_state_reason,
1083	err_event->res_slow_path_state,
1084	err_event->res_err_state_reason);
1085	} else {
1086	if (srq)
1087	event.event = IB_EVENT_QP_LAST_WQE_REACHED;
1088	}
1089
1090	if (event.event == IB_EVENT_SRQ_ERR && srq->ib_srq.event_handler) {
1091	(*srq->ib_srq.event_handler)(&event,
1092	srq->ib_srq.srq_context);
1093	} else if (event.device && qp->ib_qp.event_handler) {
1094	qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context);
1095	}
1096
1097	return 0;
1098	}
1099
1100	static int bnxt_re_handle_cq_async_error(void *event, struct bnxt_re_cq *cq)
1101	{
1102	struct creq_cq_error_notification *cqerr;
1103	struct ib_event ibevent = {};
1104
1105	cqerr = event;
1106	switch (cqerr->cq_err_reason) {
1107	case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_INVALID_ERROR:
1108	case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_OVERFLOW_ERROR:
1109	case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_LOAD_ERROR:
1110	case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_INVALID_ERROR:
1111	case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_OVERFLOW_ERROR:
1112	case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_LOAD_ERROR:
1113	ibevent.event = IB_EVENT_CQ_ERR;
1114	break;
1115	default:
1116	break;
1117	}
1118
1119	if (ibevent.event == IB_EVENT_CQ_ERR && cq->ib_cq.event_handler) {
1120	ibevent.element.cq = &cq->ib_cq;
1121	ibevent.device = &cq->rdev->ibdev;
1122
1123	ibdev_dbg(&cq->rdev->ibdev,
1124	"%s err reason %d\n", __func__, cqerr->cq_err_reason);
1125	cq->ib_cq.event_handler(&ibevent, cq->ib_cq.cq_context);
1126	}
1127
1128	return 0;
1129	}
1130
1131	static int bnxt_re_handle_affi_async_event(struct creq_qp_event *affi_async,
1132	void *obj)
1133	{
1134	struct bnxt_qplib_qp *lib_qp;
1135	struct bnxt_qplib_cq *lib_cq;
1136	struct bnxt_re_qp *qp;
1137	struct bnxt_re_cq *cq;
1138	int rc = 0;
1139	u8 event;
1140
1141	if (!obj)
1142	return rc; /* QP was already dead, still return success */
1143
1144	event = affi_async->event;
1145	switch (event) {
1146	case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION:
1147	lib_qp = obj;
1148	qp = container_of(lib_qp, struct bnxt_re_qp, qplib_qp);
1149	rc = bnxt_re_handle_qp_async_event(qp_event: affi_async, qp);
1150	break;
1151	case CREQ_QP_EVENT_EVENT_CQ_ERROR_NOTIFICATION:
1152	lib_cq = obj;
1153	cq = container_of(lib_cq, struct bnxt_re_cq, qplib_cq);
1154	rc = bnxt_re_handle_cq_async_error(event: affi_async, cq);
1155	break;
1156	default:
1157	rc = -EINVAL;
1158	}
1159	return rc;
1160	}
1161
1162	static int bnxt_re_aeq_handler(struct bnxt_qplib_rcfw *rcfw,
1163	void *aeqe, void *obj)
1164	{
1165	struct creq_qp_event *affi_async;
1166	struct creq_func_event *unaffi_async;
1167	u8 type;
1168	int rc;
1169
1170	type = ((struct creq_base *)aeqe)->type;
1171	if (type == CREQ_BASE_TYPE_FUNC_EVENT) {
1172	unaffi_async = aeqe;
1173	rc = bnxt_re_handle_unaffi_async_event(unaffi_async);
1174	} else {
1175	affi_async = aeqe;
1176	rc = bnxt_re_handle_affi_async_event(affi_async, obj);
1177	}
1178
1179	return rc;
1180	}
1181
1182	static int bnxt_re_srqn_handler(struct bnxt_qplib_nq *nq,
1183	struct bnxt_qplib_srq *handle, u8 event)
1184	{
1185	struct bnxt_re_srq *srq = container_of(handle, struct bnxt_re_srq,
1186	qplib_srq);
1187	struct ib_event ib_event;
1188
1189	ib_event.device = &srq->rdev->ibdev;
1190	ib_event.element.srq = &srq->ib_srq;
1191
1192	if (srq->ib_srq.event_handler) {
1193	if (event == NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT)
1194	ib_event.event = IB_EVENT_SRQ_LIMIT_REACHED;
1195	(*srq->ib_srq.event_handler)(&ib_event,
1196	srq->ib_srq.srq_context);
1197	}
1198	return 0;
1199	}
1200
1201	static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq,
1202	struct bnxt_qplib_cq *handle)
1203	{
1204	struct bnxt_re_cq *cq = container_of(handle, struct bnxt_re_cq,
1205	qplib_cq);
1206
1207	if (cq->ib_cq.comp_handler) {
1208	/* Lock comp_handler? */
1209	(*cq->ib_cq.comp_handler)(&cq->ib_cq, cq->ib_cq.cq_context);
1210	}
1211
1212	return 0;
1213	}
1214
1215	#define BNXT_RE_GEN_P5_PF_NQ_DB 0x10000
1216	#define BNXT_RE_GEN_P5_VF_NQ_DB 0x4000
1217	static u32 bnxt_re_get_nqdb_offset(struct bnxt_re_dev *rdev, u16 indx)
1218	{
1219	return bnxt_qplib_is_chip_gen_p5(cctx: rdev->chip_ctx) ?
1220	(rdev->is_virtfn ? BNXT_RE_GEN_P5_VF_NQ_DB :
1221	BNXT_RE_GEN_P5_PF_NQ_DB) :
1222	rdev->en_dev->msix_entries[indx].db_offset;
1223	}
1224
1225	static void bnxt_re_cleanup_res(struct bnxt_re_dev *rdev)
1226	{
1227	int i;
1228
1229	for (i = 1; i < rdev->num_msix; i++)
1230	bnxt_qplib_disable_nq(nq: &rdev->nq[i - 1]);
1231
1232	if (rdev->qplib_res.rcfw)
1233	bnxt_qplib_cleanup_res(res: &rdev->qplib_res);
1234	}
1235
1236	static int bnxt_re_init_res(struct bnxt_re_dev *rdev)
1237	{
1238	int num_vec_enabled = 0;
1239	int rc = 0, i;
1240	u32 db_offt;
1241
1242	bnxt_qplib_init_res(res: &rdev->qplib_res);
1243
1244	for (i = 1; i < rdev->num_msix ; i++) {
1245	db_offt = bnxt_re_get_nqdb_offset(rdev, indx: i);
1246	rc = bnxt_qplib_enable_nq(pdev: rdev->en_dev->pdev, nq: &rdev->nq[i - 1],
1247	nq_idx: i - 1, msix_vector: rdev->en_dev->msix_entries[i].vector,
1248	bar_reg_offset: db_offt, cqn_handler: &bnxt_re_cqn_handler,
1249	srq_handler: &bnxt_re_srqn_handler);
1250	if (rc) {
1251	ibdev_err(ibdev: &rdev->ibdev,
1252	format: "Failed to enable NQ with rc = 0x%x", rc);
1253	goto fail;
1254	}
1255	num_vec_enabled++;
1256	}
1257	return 0;
1258	fail:
1259	for (i = num_vec_enabled; i >= 0; i--)
1260	bnxt_qplib_disable_nq(nq: &rdev->nq[i]);
1261	return rc;
1262	}
1263
1264	static void bnxt_re_free_nq_res(struct bnxt_re_dev *rdev)
1265	{
1266	u8 type;
1267	int i;
1268
1269	for (i = 0; i < rdev->num_msix - 1; i++) {
1270	type = bnxt_qplib_get_ring_type(cctx: rdev->chip_ctx);
1271	bnxt_re_net_ring_free(rdev, fw_ring_id: rdev->nq[i].ring_id, type);
1272	bnxt_qplib_free_nq(nq: &rdev->nq[i]);
1273	rdev->nq[i].res = NULL;
1274	}
1275	}
1276
1277	static void bnxt_re_free_res(struct bnxt_re_dev *rdev)
1278	{
1279	bnxt_re_free_nq_res(rdev);
1280
1281	if (rdev->qplib_res.dpi_tbl.max) {
1282	bnxt_qplib_dealloc_dpi(res: &rdev->qplib_res,
1283	dpi: &rdev->dpi_privileged);
1284	}
1285	if (rdev->qplib_res.rcfw) {
1286	bnxt_qplib_free_res(res: &rdev->qplib_res);
1287	rdev->qplib_res.rcfw = NULL;
1288	}
1289	}
1290
1291	static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev)
1292	{
1293	struct bnxt_re_ring_attr rattr = {};
1294	int num_vec_created = 0;
1295	int rc, i;
1296	u8 type;
1297
1298	/* Configure and allocate resources for qplib */
1299	rdev->qplib_res.rcfw = &rdev->rcfw;
1300	rc = bnxt_qplib_get_dev_attr(rcfw: &rdev->rcfw, attr: &rdev->dev_attr);
1301	if (rc)
1302	goto fail;
1303
1304	rc = bnxt_qplib_alloc_res(res: &rdev->qplib_res, pdev: rdev->en_dev->pdev,
1305	netdev: rdev->netdev, dev_attr: &rdev->dev_attr);
1306	if (rc)
1307	goto fail;
1308
1309	rc = bnxt_qplib_alloc_dpi(res: &rdev->qplib_res,
1310	dpi: &rdev->dpi_privileged,
1311	app: rdev, type: BNXT_QPLIB_DPI_TYPE_KERNEL);
1312	if (rc)
1313	goto dealloc_res;
1314
1315	for (i = 0; i < rdev->num_msix - 1; i++) {
1316	struct bnxt_qplib_nq *nq;
1317
1318	nq = &rdev->nq[i];
1319	nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT;
1320	rc = bnxt_qplib_alloc_nq(res: &rdev->qplib_res, nq: &rdev->nq[i]);
1321	if (rc) {
1322	ibdev_err(ibdev: &rdev->ibdev, format: "Alloc Failed NQ%d rc:%#x",
1323	i, rc);
1324	goto free_nq;
1325	}
1326	type = bnxt_qplib_get_ring_type(cctx: rdev->chip_ctx);
1327	rattr.dma_arr = nq->hwq.pbl[PBL_LVL_0].pg_map_arr;
1328	rattr.pages = nq->hwq.pbl[rdev->nq[i].hwq.level].pg_count;
1329	rattr.type = type;
1330	rattr.mode = RING_ALLOC_REQ_INT_MODE_MSIX;
1331	rattr.depth = BNXT_QPLIB_NQE_MAX_CNT - 1;
1332	rattr.lrid = rdev->en_dev->msix_entries[i + 1].ring_idx;
1333	rc = bnxt_re_net_ring_alloc(rdev, ring_attr: &rattr, fw_ring_id: &nq->ring_id);
1334	if (rc) {
1335	ibdev_err(ibdev: &rdev->ibdev,
1336	format: "Failed to allocate NQ fw id with rc = 0x%x",
1337	rc);
1338	bnxt_qplib_free_nq(nq: &rdev->nq[i]);
1339	goto free_nq;
1340	}
1341	num_vec_created++;
1342	}
1343	return 0;
1344	free_nq:
1345	for (i = num_vec_created - 1; i >= 0; i--) {
1346	type = bnxt_qplib_get_ring_type(cctx: rdev->chip_ctx);
1347	bnxt_re_net_ring_free(rdev, fw_ring_id: rdev->nq[i].ring_id, type);
1348	bnxt_qplib_free_nq(nq: &rdev->nq[i]);
1349	}
1350	bnxt_qplib_dealloc_dpi(res: &rdev->qplib_res,
1351	dpi: &rdev->dpi_privileged);
1352	dealloc_res:
1353	bnxt_qplib_free_res(res: &rdev->qplib_res);
1354
1355	fail:
1356	rdev->qplib_res.rcfw = NULL;
1357	return rc;
1358	}
1359
1360	static void bnxt_re_dispatch_event(struct ib_device *ibdev, struct ib_qp *qp,
1361	u8 port_num, enum ib_event_type event)
1362	{
1363	struct ib_event ib_event;
1364
1365	ib_event.device = ibdev;
1366	if (qp) {
1367	ib_event.element.qp = qp;
1368	ib_event.event = event;
1369	if (qp->event_handler)
1370	qp->event_handler(&ib_event, qp->qp_context);
1371
1372	} else {
1373	ib_event.element.port_num = port_num;
1374	ib_event.event = event;
1375	ib_dispatch_event(event: &ib_event);
1376	}
1377	}
1378
1379	static bool bnxt_re_is_qp1_or_shadow_qp(struct bnxt_re_dev *rdev,
1380	struct bnxt_re_qp *qp)
1381	{
1382	return (qp->ib_qp.qp_type == IB_QPT_GSI) ||
1383	(qp == rdev->gsi_ctx.gsi_sqp);
1384	}
1385
1386	static void bnxt_re_dev_stop(struct bnxt_re_dev *rdev)
1387	{
1388	int mask = IB_QP_STATE;
1389	struct ib_qp_attr qp_attr;
1390	struct bnxt_re_qp *qp;
1391
1392	qp_attr.qp_state = IB_QPS_ERR;
1393	mutex_lock(&rdev->qp_lock);
1394	list_for_each_entry(qp, &rdev->qp_list, list) {
1395	/* Modify the state of all QPs except QP1/Shadow QP */
1396	if (!bnxt_re_is_qp1_or_shadow_qp(rdev, qp)) {
1397	if (qp->qplib_qp.state !=
1398	CMDQ_MODIFY_QP_NEW_STATE_RESET &&
1399	qp->qplib_qp.state !=
1400	CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1401	bnxt_re_dispatch_event(ibdev: &rdev->ibdev, qp: &qp->ib_qp,
1402	port_num: 1, event: IB_EVENT_QP_FATAL);
1403	bnxt_re_modify_qp(qp: &qp->ib_qp, qp_attr: &qp_attr, qp_attr_mask: mask,
1404	NULL);
1405	}
1406	}
1407	}
1408	mutex_unlock(lock: &rdev->qp_lock);
1409	}
1410
1411	static int bnxt_re_update_gid(struct bnxt_re_dev *rdev)
1412	{
1413	struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
1414	struct bnxt_qplib_gid gid;
1415	u16 gid_idx, index;
1416	int rc = 0;
1417
1418	if (!ib_device_try_get(dev: &rdev->ibdev))
1419	return 0;
1420
1421	for (index = 0; index < sgid_tbl->active; index++) {
1422	gid_idx = sgid_tbl->hw_id[index];
1423
1424	if (!memcmp(p: &sgid_tbl->tbl[index], q: &bnxt_qplib_gid_zero,
1425	size: sizeof(bnxt_qplib_gid_zero)))
1426	continue;
1427	/* need to modify the VLAN enable setting of non VLAN GID only
1428	* as setting is done for VLAN GID while adding GID
1429	*/
1430	if (sgid_tbl->vlan[index])
1431	continue;
1432
1433	memcpy(&gid, &sgid_tbl->tbl[index], sizeof(gid));
1434
1435	rc = bnxt_qplib_update_sgid(sgid_tbl, gid: &gid, gid_idx,
1436	smac: rdev->qplib_res.netdev->dev_addr);
1437	}
1438
1439	ib_device_put(device: &rdev->ibdev);
1440	return rc;
1441	}
1442
1443	static u32 bnxt_re_get_priority_mask(struct bnxt_re_dev *rdev)
1444	{
1445	u32 prio_map = 0, tmp_map = 0;
1446	struct net_device *netdev;
1447	struct dcb_app app = {};
1448
1449	netdev = rdev->netdev;
1450
1451	app.selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE;
1452	app.protocol = ETH_P_IBOE;
1453	tmp_map = dcb_ieee_getapp_mask(netdev, &app);
1454	prio_map = tmp_map;
1455
1456	app.selector = IEEE_8021QAZ_APP_SEL_DGRAM;
1457	app.protocol = ROCE_V2_UDP_DPORT;
1458	tmp_map = dcb_ieee_getapp_mask(netdev, &app);
1459	prio_map |= tmp_map;
1460
1461	return prio_map;
1462	}
1463
1464	static int bnxt_re_setup_qos(struct bnxt_re_dev *rdev)
1465	{
1466	u8 prio_map = 0;
1467
1468	/* Get priority for roce */
1469	prio_map = bnxt_re_get_priority_mask(rdev);
1470
1471	if (prio_map == rdev->cur_prio_map)
1472	return 0;
1473	rdev->cur_prio_map = prio_map;
1474	/* Actual priorities are not programmed as they are already
1475	* done by L2 driver; just enable or disable priority vlan tagging
1476	*/
1477	if ((prio_map == 0 && rdev->qplib_res.prio) ||
1478	(prio_map != 0 && !rdev->qplib_res.prio)) {
1479	rdev->qplib_res.prio = prio_map;
1480	bnxt_re_update_gid(rdev);
1481	}
1482
1483	return 0;
1484	}
1485
1486	static void bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev)
1487	{
1488	struct bnxt_en_dev *en_dev = rdev->en_dev;
1489	struct hwrm_ver_get_output resp = {};
1490	struct hwrm_ver_get_input req = {};
1491	struct bnxt_qplib_chip_ctx *cctx;
1492	struct bnxt_fw_msg fw_msg = {};
1493	int rc;
1494
1495	bnxt_re_init_hwrm_hdr(hdr: (void *)&req, opcd: HWRM_VER_GET);
1496	req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
1497	req.hwrm_intf_min = HWRM_VERSION_MINOR;
1498	req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
1499	bnxt_re_fill_fw_msg(fw_msg: &fw_msg, msg: (void *)&req, msg_len: sizeof(req), resp: (void *)&resp,
1500	resp_max_len: sizeof(resp), timeout: DFLT_HWRM_CMD_TIMEOUT);
1501	rc = bnxt_send_msg(en_dev, &fw_msg);
1502	if (rc) {
1503	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to query HW version, rc = 0x%x",
1504	rc);
1505	return;
1506	}
1507
1508	cctx = rdev->chip_ctx;
1509	cctx->hwrm_intf_ver =
1510	(u64)le16_to_cpu(resp.hwrm_intf_major) << 48 |
1511	(u64)le16_to_cpu(resp.hwrm_intf_minor) << 32 |
1512	(u64)le16_to_cpu(resp.hwrm_intf_build) << 16 |
1513	le16_to_cpu(resp.hwrm_intf_patch);
1514
1515	cctx->hwrm_cmd_max_timeout = le16_to_cpu(resp.max_req_timeout);
1516
1517	if (!cctx->hwrm_cmd_max_timeout)
1518	cctx->hwrm_cmd_max_timeout = RCFW_FW_STALL_MAX_TIMEOUT;
1519	}
1520
1521	static int bnxt_re_ib_init(struct bnxt_re_dev *rdev)
1522	{
1523	int rc;
1524	u32 event;
1525
1526	/* Register ib dev */
1527	rc = bnxt_re_register_ib(rdev);
1528	if (rc) {
1529	pr_err("Failed to register with IB: %#x\n", rc);
1530	return rc;
1531	}
1532	dev_info(rdev_to_dev(rdev), "Device registered with IB successfully");
1533	set_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, addr: &rdev->flags);
1534
1535	event = netif_running(dev: rdev->netdev) && netif_carrier_ok(dev: rdev->netdev) ?
1536	IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
1537
1538	bnxt_re_dispatch_event(ibdev: &rdev->ibdev, NULL, port_num: 1, event);
1539
1540	return rc;
1541	}
1542
1543	static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev)
1544	{
1545	u8 type;
1546	int rc;
1547
1548	if (test_and_clear_bit(BNXT_RE_FLAG_QOS_WORK_REG, addr: &rdev->flags))
1549	cancel_delayed_work_sync(dwork: &rdev->worker);
1550
1551	if (test_and_clear_bit(BNXT_RE_FLAG_RESOURCES_INITIALIZED,
1552	addr: &rdev->flags))
1553	bnxt_re_cleanup_res(rdev);
1554	if (test_and_clear_bit(BNXT_RE_FLAG_RESOURCES_ALLOCATED, addr: &rdev->flags))
1555	bnxt_re_free_res(rdev);
1556
1557	if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, addr: &rdev->flags)) {
1558	rc = bnxt_qplib_deinit_rcfw(rcfw: &rdev->rcfw);
1559	if (rc)
1560	ibdev_warn(ibdev: &rdev->ibdev,
1561	format: "Failed to deinitialize RCFW: %#x", rc);
1562	bnxt_re_net_stats_ctx_free(rdev, fw_stats_ctx_id: rdev->qplib_ctx.stats.fw_id);
1563	bnxt_qplib_free_ctx(res: &rdev->qplib_res, ctx: &rdev->qplib_ctx);
1564	bnxt_qplib_disable_rcfw_channel(rcfw: &rdev->rcfw);
1565	type = bnxt_qplib_get_ring_type(cctx: rdev->chip_ctx);
1566	bnxt_re_net_ring_free(rdev, fw_ring_id: rdev->rcfw.creq.ring_id, type);
1567	bnxt_qplib_free_rcfw_channel(rcfw: &rdev->rcfw);
1568	}
1569
1570	rdev->num_msix = 0;
1571
1572	if (rdev->pacing.dbr_pacing)
1573	bnxt_re_deinitialize_dbr_pacing(rdev);
1574
1575	bnxt_re_destroy_chip_ctx(rdev);
1576	if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, addr: &rdev->flags))
1577	bnxt_unregister_dev(rdev->en_dev);
1578	}
1579
1580	/* worker thread for polling periodic events. Now used for QoS programming*/
1581	static void bnxt_re_worker(struct work_struct *work)
1582	{
1583	struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev,
1584	worker.work);
1585
1586	bnxt_re_setup_qos(rdev);
1587	schedule_delayed_work(dwork: &rdev->worker, delay: msecs_to_jiffies(m: 30000));
1588	}
1589
1590	static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 wqe_mode)
1591	{
1592	struct bnxt_re_ring_attr rattr = {};
1593	struct bnxt_qplib_creq_ctx *creq;
1594	u32 db_offt;
1595	int vid;
1596	u8 type;
1597	int rc;
1598
1599	/* Registered a new RoCE device instance to netdev */
1600	rc = bnxt_re_register_netdev(rdev);
1601	if (rc) {
1602	ibdev_err(ibdev: &rdev->ibdev,
1603	format: "Failed to register with netedev: %#x\n", rc);
1604	return -EINVAL;
1605	}
1606	set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, addr: &rdev->flags);
1607
1608	rc = bnxt_re_setup_chip_ctx(rdev, wqe_mode);
1609	if (rc) {
1610	bnxt_unregister_dev(rdev->en_dev);
1611	clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, addr: &rdev->flags);
1612	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to get chip context\n");
1613	return -EINVAL;
1614	}
1615
1616	/* Check whether VF or PF */
1617	bnxt_re_get_sriov_func_type(rdev);
1618
1619	if (!rdev->en_dev->ulp_tbl->msix_requested) {
1620	ibdev_err(ibdev: &rdev->ibdev,
1621	format: "Failed to get MSI-X vectors: %#x\n", rc);
1622	rc = -EINVAL;
1623	goto fail;
1624	}
1625	ibdev_dbg(&rdev->ibdev, "Got %d MSI-X vectors\n",
1626	rdev->en_dev->ulp_tbl->msix_requested);
1627	rdev->num_msix = rdev->en_dev->ulp_tbl->msix_requested;
1628
1629	bnxt_re_query_hwrm_intf_version(rdev);
1630
1631	/* Establish RCFW Communication Channel to initialize the context
1632	* memory for the function and all child VFs
1633	*/
1634	rc = bnxt_qplib_alloc_rcfw_channel(res: &rdev->qplib_res, rcfw: &rdev->rcfw,
1635	ctx: &rdev->qplib_ctx,
1636	BNXT_RE_MAX_QPC_COUNT);
1637	if (rc) {
1638	ibdev_err(ibdev: &rdev->ibdev,
1639	format: "Failed to allocate RCFW Channel: %#x\n", rc);
1640	goto fail;
1641	}
1642
1643	type = bnxt_qplib_get_ring_type(cctx: rdev->chip_ctx);
1644	creq = &rdev->rcfw.creq;
1645	rattr.dma_arr = creq->hwq.pbl[PBL_LVL_0].pg_map_arr;
1646	rattr.pages = creq->hwq.pbl[creq->hwq.level].pg_count;
1647	rattr.type = type;
1648	rattr.mode = RING_ALLOC_REQ_INT_MODE_MSIX;
1649	rattr.depth = BNXT_QPLIB_CREQE_MAX_CNT - 1;
1650	rattr.lrid = rdev->en_dev->msix_entries[BNXT_RE_AEQ_IDX].ring_idx;
1651	rc = bnxt_re_net_ring_alloc(rdev, ring_attr: &rattr, fw_ring_id: &creq->ring_id);
1652	if (rc) {
1653	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to allocate CREQ: %#x\n", rc);
1654	goto free_rcfw;
1655	}
1656	db_offt = bnxt_re_get_nqdb_offset(rdev, BNXT_RE_AEQ_IDX);
1657	vid = rdev->en_dev->msix_entries[BNXT_RE_AEQ_IDX].vector;
1658	rc = bnxt_qplib_enable_rcfw_channel(rcfw: &rdev->rcfw,
1659	msix_vector: vid, cp_bar_reg_off: db_offt,
1660	aeq_handler: &bnxt_re_aeq_handler);
1661	if (rc) {
1662	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to enable RCFW channel: %#x\n",
1663	rc);
1664	goto free_ring;
1665	}
1666
1667	if (bnxt_qplib_dbr_pacing_en(cctx: rdev->chip_ctx)) {
1668	rc = bnxt_re_initialize_dbr_pacing(rdev);
1669	if (!rc) {
1670	rdev->pacing.dbr_pacing = true;
1671	} else {
1672	ibdev_err(ibdev: &rdev->ibdev,
1673	format: "DBR pacing disabled with error : %d\n", rc);
1674	rdev->pacing.dbr_pacing = false;
1675	}
1676	}
1677	rc = bnxt_qplib_get_dev_attr(rcfw: &rdev->rcfw, attr: &rdev->dev_attr);
1678	if (rc)
1679	goto disable_rcfw;
1680
1681	bnxt_re_set_resource_limits(rdev);
1682
1683	rc = bnxt_qplib_alloc_ctx(res: &rdev->qplib_res, ctx: &rdev->qplib_ctx, virt_fn: 0,
1684	is_p5: bnxt_qplib_is_chip_gen_p5(cctx: rdev->chip_ctx));
1685	if (rc) {
1686	ibdev_err(ibdev: &rdev->ibdev,
1687	format: "Failed to allocate QPLIB context: %#x\n", rc);
1688	goto disable_rcfw;
1689	}
1690	rc = bnxt_re_net_stats_ctx_alloc(rdev,
1691	dma_map: rdev->qplib_ctx.stats.dma_map,
1692	fw_stats_ctx_id: &rdev->qplib_ctx.stats.fw_id);
1693	if (rc) {
1694	ibdev_err(ibdev: &rdev->ibdev,
1695	format: "Failed to allocate stats context: %#x\n", rc);
1696	goto free_ctx;
1697	}
1698
1699	rc = bnxt_qplib_init_rcfw(rcfw: &rdev->rcfw, ctx: &rdev->qplib_ctx,
1700	is_virtfn: rdev->is_virtfn);
1701	if (rc) {
1702	ibdev_err(ibdev: &rdev->ibdev,
1703	format: "Failed to initialize RCFW: %#x\n", rc);
1704	goto free_sctx;
1705	}
1706	set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, addr: &rdev->flags);
1707
1708	/* Resources based on the 'new' device caps */
1709	rc = bnxt_re_alloc_res(rdev);
1710	if (rc) {
1711	ibdev_err(ibdev: &rdev->ibdev,
1712	format: "Failed to allocate resources: %#x\n", rc);
1713	goto fail;
1714	}
1715	set_bit(BNXT_RE_FLAG_RESOURCES_ALLOCATED, addr: &rdev->flags);
1716	rc = bnxt_re_init_res(rdev);
1717	if (rc) {
1718	ibdev_err(ibdev: &rdev->ibdev,
1719	format: "Failed to initialize resources: %#x\n", rc);
1720	goto fail;
1721	}
1722
1723	set_bit(BNXT_RE_FLAG_RESOURCES_INITIALIZED, addr: &rdev->flags);
1724
1725	if (!rdev->is_virtfn) {
1726	rc = bnxt_re_setup_qos(rdev);
1727	if (rc)
1728	ibdev_info(ibdev: &rdev->ibdev,
1729	format: "RoCE priority not yet configured\n");
1730
1731	INIT_DELAYED_WORK(&rdev->worker, bnxt_re_worker);
1732	set_bit(BNXT_RE_FLAG_QOS_WORK_REG, addr: &rdev->flags);
1733	schedule_delayed_work(dwork: &rdev->worker, delay: msecs_to_jiffies(m: 30000));
1734	/*
1735	* Use the total VF count since the actual VF count may not be
1736	* available at this point.
1737	*/
1738	bnxt_re_vf_res_config(rdev);
1739	}
1740
1741	return 0;
1742	free_sctx:
1743	bnxt_re_net_stats_ctx_free(rdev, fw_stats_ctx_id: rdev->qplib_ctx.stats.fw_id);
1744	free_ctx:
1745	bnxt_qplib_free_ctx(res: &rdev->qplib_res, ctx: &rdev->qplib_ctx);
1746	disable_rcfw:
1747	bnxt_qplib_disable_rcfw_channel(rcfw: &rdev->rcfw);
1748	free_ring:
1749	type = bnxt_qplib_get_ring_type(cctx: rdev->chip_ctx);
1750	bnxt_re_net_ring_free(rdev, fw_ring_id: rdev->rcfw.creq.ring_id, type);
1751	free_rcfw:
1752	bnxt_qplib_free_rcfw_channel(rcfw: &rdev->rcfw);
1753	fail:
1754	bnxt_re_dev_uninit(rdev);
1755
1756	return rc;
1757	}
1758
1759	static int bnxt_re_add_device(struct auxiliary_device *adev, u8 wqe_mode)
1760	{
1761	struct bnxt_aux_priv *aux_priv =
1762	container_of(adev, struct bnxt_aux_priv, aux_dev);
1763	struct bnxt_en_dev *en_dev;
1764	struct bnxt_re_dev *rdev;
1765	int rc;
1766
1767	/* en_dev should never be NULL as long as adev and aux_dev are valid. */
1768	en_dev = aux_priv->edev;
1769
1770	rdev = bnxt_re_dev_add(aux_priv, en_dev);
1771	if (!rdev || !rdev_to_dev(rdev)) {
1772	rc = -ENOMEM;
1773	goto exit;
1774	}
1775
1776	rc = bnxt_re_dev_init(rdev, wqe_mode);
1777	if (rc)
1778	goto re_dev_dealloc;
1779
1780	rc = bnxt_re_ib_init(rdev);
1781	if (rc) {
1782	pr_err("Failed to register with IB: %s",
1783	aux_priv->aux_dev.name);
1784	goto re_dev_uninit;
1785	}
1786	auxiliary_set_drvdata(auxdev: adev, data: rdev);
1787
1788	return 0;
1789
1790	re_dev_uninit:
1791	bnxt_re_dev_uninit(rdev);
1792	re_dev_dealloc:
1793	ib_dealloc_device(device: &rdev->ibdev);
1794	exit:
1795	return rc;
1796	}
1797
1798	static void bnxt_re_setup_cc(struct bnxt_re_dev *rdev, bool enable)
1799	{
1800	struct bnxt_qplib_cc_param cc_param = {};
1801
1802	/* Do not enable congestion control on VFs */
1803	if (rdev->is_virtfn)
1804	return;
1805
1806	/* Currently enabling only for GenP5 adapters */
1807	if (!bnxt_qplib_is_chip_gen_p5(cctx: rdev->chip_ctx))
1808	return;
1809
1810	if (enable) {
1811	cc_param.enable = 1;
1812	cc_param.cc_mode = CMDQ_MODIFY_ROCE_CC_CC_MODE_PROBABILISTIC_CC_MODE;
1813	}
1814
1815	cc_param.mask = (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE |
1816	CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC |
1817	CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN);
1818
1819	if (bnxt_qplib_modify_cc(res: &rdev->qplib_res, cc_param: &cc_param))
1820	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to setup CC enable = %d\n", enable);
1821	}
1822
1823	/*
1824	* "Notifier chain callback can be invoked for the same chain from
1825	* different CPUs at the same time".
1826	*
1827	* For cases when the netdev is already present, our call to the
1828	* register_netdevice_notifier() will actually get the rtnl_lock()
1829	* before sending NETDEV_REGISTER and (if up) NETDEV_UP
1830	* events.
1831	*
1832	* But for cases when the netdev is not already present, the notifier
1833	* chain is subjected to be invoked from different CPUs simultaneously.
1834	*
1835	* This is protected by the netdev_mutex.
1836	*/
1837	static int bnxt_re_netdev_event(struct notifier_block *notifier,
1838	unsigned long event, void *ptr)
1839	{
1840	struct net_device *real_dev, *netdev = netdev_notifier_info_to_dev(info: ptr);
1841	struct bnxt_re_dev *rdev;
1842
1843	real_dev = rdma_vlan_dev_real_dev(dev: netdev);
1844	if (!real_dev)
1845	real_dev = netdev;
1846
1847	if (real_dev != netdev)
1848	goto exit;
1849
1850	rdev = bnxt_re_from_netdev(netdev: real_dev);
1851	if (!rdev)
1852	return NOTIFY_DONE;
1853
1854
1855	switch (event) {
1856	case NETDEV_UP:
1857	case NETDEV_DOWN:
1858	case NETDEV_CHANGE:
1859	bnxt_re_dispatch_event(ibdev: &rdev->ibdev, NULL, port_num: 1,
1860	event: netif_carrier_ok(dev: real_dev) ?
1861	IB_EVENT_PORT_ACTIVE :
1862	IB_EVENT_PORT_ERR);
1863	break;
1864	default:
1865	break;
1866	}
1867	ib_device_put(device: &rdev->ibdev);
1868	exit:
1869	return NOTIFY_DONE;
1870	}
1871
1872	#define BNXT_ADEV_NAME "bnxt_en"
1873
1874	static void bnxt_re_remove(struct auxiliary_device *adev)
1875	{
1876	struct bnxt_re_dev *rdev = auxiliary_get_drvdata(auxdev: adev);
1877
1878	if (!rdev)
1879	return;
1880
1881	mutex_lock(&bnxt_re_mutex);
1882	if (rdev->nb.notifier_call) {
1883	unregister_netdevice_notifier(nb: &rdev->nb);
1884	rdev->nb.notifier_call = NULL;
1885	} else {
1886	/* If notifier is null, we should have already done a
1887	* clean up before coming here.
1888	*/
1889	goto skip_remove;
1890	}
1891	bnxt_re_setup_cc(rdev, enable: false);
1892	ib_unregister_device(device: &rdev->ibdev);
1893	bnxt_re_dev_uninit(rdev);
1894	ib_dealloc_device(device: &rdev->ibdev);
1895	skip_remove:
1896	mutex_unlock(lock: &bnxt_re_mutex);
1897	}
1898
1899	static int bnxt_re_probe(struct auxiliary_device *adev,
1900	const struct auxiliary_device_id *id)
1901	{
1902	struct bnxt_re_dev *rdev;
1903	int rc;
1904
1905	mutex_lock(&bnxt_re_mutex);
1906	rc = bnxt_re_add_device(adev, wqe_mode: BNXT_QPLIB_WQE_MODE_STATIC);
1907	if (rc) {
1908	mutex_unlock(lock: &bnxt_re_mutex);
1909	return rc;
1910	}
1911
1912	rdev = auxiliary_get_drvdata(auxdev: adev);
1913
1914	rdev->nb.notifier_call = bnxt_re_netdev_event;
1915	rc = register_netdevice_notifier(nb: &rdev->nb);
1916	if (rc) {
1917	rdev->nb.notifier_call = NULL;
1918	pr_err("%s: Cannot register to netdevice_notifier",
1919	ROCE_DRV_MODULE_NAME);
1920	goto err;
1921	}
1922
1923	bnxt_re_setup_cc(rdev, enable: true);
1924	mutex_unlock(lock: &bnxt_re_mutex);
1925	return 0;
1926
1927	err:
1928	mutex_unlock(lock: &bnxt_re_mutex);
1929	bnxt_re_remove(adev);
1930
1931	return rc;
1932	}
1933
1934	static int bnxt_re_suspend(struct auxiliary_device *adev, pm_message_t state)
1935	{
1936	struct bnxt_re_dev *rdev = auxiliary_get_drvdata(auxdev: adev);
1937
1938	if (!rdev)
1939	return 0;
1940
1941	mutex_lock(&bnxt_re_mutex);
1942	/* L2 driver may invoke this callback during device error/crash or device
1943	* reset. Current RoCE driver doesn't recover the device in case of
1944	* error. Handle the error by dispatching fatal events to all qps
1945	* ie. by calling bnxt_re_dev_stop and release the MSIx vectors as
1946	* L2 driver want to modify the MSIx table.
1947	*/
1948
1949	ibdev_info(ibdev: &rdev->ibdev, format: "Handle device suspend call");
1950	/* Check the current device state from bnxt_en_dev and move the
1951	* device to detached state if FW_FATAL_COND is set.
1952	* This prevents more commands to HW during clean-up,
1953	* in case the device is already in error.
1954	*/
1955	if (test_bit(BNXT_STATE_FW_FATAL_COND, &rdev->en_dev->en_state))
1956	set_bit(ERR_DEVICE_DETACHED, addr: &rdev->rcfw.cmdq.flags);
1957
1958	bnxt_re_dev_stop(rdev);
1959	bnxt_re_stop_irq(handle: rdev);
1960	/* Move the device states to detached and avoid sending any more
1961	* commands to HW
1962	*/
1963	set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, addr: &rdev->flags);
1964	set_bit(ERR_DEVICE_DETACHED, addr: &rdev->rcfw.cmdq.flags);
1965	wake_up_all(&rdev->rcfw.cmdq.waitq);
1966	mutex_unlock(lock: &bnxt_re_mutex);
1967
1968	return 0;
1969	}
1970
1971	static int bnxt_re_resume(struct auxiliary_device *adev)
1972	{
1973	struct bnxt_re_dev *rdev = auxiliary_get_drvdata(auxdev: adev);
1974
1975	if (!rdev)
1976	return 0;
1977
1978	mutex_lock(&bnxt_re_mutex);
1979	/* L2 driver may invoke this callback during device recovery, resume.
1980	* reset. Current RoCE driver doesn't recover the device in case of
1981	* error. Handle the error by dispatching fatal events to all qps
1982	* ie. by calling bnxt_re_dev_stop and release the MSIx vectors as
1983	* L2 driver want to modify the MSIx table.
1984	*/
1985
1986	ibdev_info(ibdev: &rdev->ibdev, format: "Handle device resume call");
1987	mutex_unlock(lock: &bnxt_re_mutex);
1988
1989	return 0;
1990	}
1991
1992	static const struct auxiliary_device_id bnxt_re_id_table[] = {
1993	{ .name = BNXT_ADEV_NAME ".rdma", },
1994	{},
1995	};
1996
1997	MODULE_DEVICE_TABLE(auxiliary, bnxt_re_id_table);
1998
1999	static struct auxiliary_driver bnxt_re_driver = {
2000	.name = "rdma",
2001	.probe = bnxt_re_probe,
2002	.remove = bnxt_re_remove,
2003	.shutdown = bnxt_re_shutdown,
2004	.suspend = bnxt_re_suspend,
2005	.resume = bnxt_re_resume,
2006	.id_table = bnxt_re_id_table,
2007	};
2008
2009	static int __init bnxt_re_mod_init(void)
2010	{
2011	int rc;
2012
2013	pr_info("%s: %s", ROCE_DRV_MODULE_NAME, version);
2014	rc = auxiliary_driver_register(&bnxt_re_driver);
2015	if (rc) {
2016	pr_err("%s: Failed to register auxiliary driver\n",
2017	ROCE_DRV_MODULE_NAME);
2018	return rc;
2019	}
2020	return 0;
2021	}
2022
2023	static void __exit bnxt_re_mod_exit(void)
2024	{
2025	auxiliary_driver_unregister(auxdrv: &bnxt_re_driver);
2026	}
2027
2028	module_init(bnxt_re_mod_init);
2029	module_exit(bnxt_re_mod_exit);
2030