1 | /* Broadcom NetXtreme-C/E network driver. |
2 | * |
3 | * Copyright (c) 2016-2018 Broadcom Limited |
4 | * |
5 | * This program is free software; you can redistribute it and/or modify |
6 | * it under the terms of the GNU General Public License as published by |
7 | * the Free Software Foundation. |
8 | */ |
9 | |
10 | #include <linux/module.h> |
11 | |
12 | #include <linux/kernel.h> |
13 | #include <linux/errno.h> |
14 | #include <linux/interrupt.h> |
15 | #include <linux/pci.h> |
16 | #include <linux/netdevice.h> |
17 | #include <linux/rtnetlink.h> |
18 | #include <linux/bitops.h> |
19 | #include <linux/irq.h> |
20 | #include <asm/byteorder.h> |
21 | #include <linux/bitmap.h> |
22 | #include <linux/auxiliary_bus.h> |
23 | |
24 | #include "bnxt_hsi.h" |
25 | #include "bnxt.h" |
26 | #include "bnxt_hwrm.h" |
27 | #include "bnxt_ulp.h" |
28 | |
29 | static DEFINE_IDA(bnxt_aux_dev_ids); |
30 | |
31 | static void bnxt_fill_msix_vecs(struct bnxt *bp, struct bnxt_msix_entry *ent) |
32 | { |
33 | struct bnxt_en_dev *edev = bp->edev; |
34 | int num_msix, idx, i; |
35 | |
36 | if (!edev->ulp_tbl->msix_requested) { |
37 | netdev_warn(dev: bp->dev, format: "Requested MSI-X vectors insufficient\n" ); |
38 | return; |
39 | } |
40 | num_msix = edev->ulp_tbl->msix_requested; |
41 | idx = edev->ulp_tbl->msix_base; |
42 | for (i = 0; i < num_msix; i++) { |
43 | ent[i].vector = bp->irq_tbl[idx + i].vector; |
44 | ent[i].ring_idx = idx + i; |
45 | if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) |
46 | ent[i].db_offset = bp->db_offset; |
47 | else |
48 | ent[i].db_offset = (idx + i) * 0x80; |
49 | } |
50 | } |
51 | |
52 | int bnxt_register_dev(struct bnxt_en_dev *edev, |
53 | struct bnxt_ulp_ops *ulp_ops, |
54 | void *handle) |
55 | { |
56 | struct net_device *dev = edev->net; |
57 | struct bnxt *bp = netdev_priv(dev); |
58 | unsigned int max_stat_ctxs; |
59 | struct bnxt_ulp *ulp; |
60 | |
61 | max_stat_ctxs = bnxt_get_max_func_stat_ctxs(bp); |
62 | if (max_stat_ctxs <= BNXT_MIN_ROCE_STAT_CTXS || |
63 | bp->cp_nr_rings == max_stat_ctxs) |
64 | return -ENOMEM; |
65 | |
66 | ulp = edev->ulp_tbl; |
67 | if (!ulp) |
68 | return -ENOMEM; |
69 | |
70 | ulp->handle = handle; |
71 | rcu_assign_pointer(ulp->ulp_ops, ulp_ops); |
72 | |
73 | if (test_bit(BNXT_STATE_OPEN, &bp->state)) |
74 | bnxt_hwrm_vnic_cfg(bp, vnic_id: 0); |
75 | |
76 | bnxt_fill_msix_vecs(bp, ent: bp->edev->msix_entries); |
77 | edev->flags |= BNXT_EN_FLAG_MSIX_REQUESTED; |
78 | return 0; |
79 | } |
80 | EXPORT_SYMBOL(bnxt_register_dev); |
81 | |
82 | void bnxt_unregister_dev(struct bnxt_en_dev *edev) |
83 | { |
84 | struct net_device *dev = edev->net; |
85 | struct bnxt *bp = netdev_priv(dev); |
86 | struct bnxt_ulp *ulp; |
87 | int i = 0; |
88 | |
89 | ulp = edev->ulp_tbl; |
90 | if (ulp->msix_requested) |
91 | edev->flags &= ~BNXT_EN_FLAG_MSIX_REQUESTED; |
92 | |
93 | if (ulp->max_async_event_id) |
94 | bnxt_hwrm_func_drv_rgtr(bp, NULL, bmap_size: 0, async_only: true); |
95 | |
96 | RCU_INIT_POINTER(ulp->ulp_ops, NULL); |
97 | synchronize_rcu(); |
98 | ulp->max_async_event_id = 0; |
99 | ulp->async_events_bmap = NULL; |
100 | while (atomic_read(v: &ulp->ref_count) != 0 && i < 10) { |
101 | msleep(msecs: 100); |
102 | i++; |
103 | } |
104 | return; |
105 | } |
106 | EXPORT_SYMBOL(bnxt_unregister_dev); |
107 | |
108 | int bnxt_get_ulp_msix_num(struct bnxt *bp) |
109 | { |
110 | u32 roce_msix = BNXT_VF(bp) ? |
111 | BNXT_MAX_VF_ROCE_MSIX : BNXT_MAX_ROCE_MSIX; |
112 | |
113 | return ((bp->flags & BNXT_FLAG_ROCE_CAP) ? |
114 | min_t(u32, roce_msix, num_online_cpus()) : 0); |
115 | } |
116 | |
117 | int bnxt_get_ulp_msix_base(struct bnxt *bp) |
118 | { |
119 | if (bnxt_ulp_registered(edev: bp->edev)) { |
120 | struct bnxt_en_dev *edev = bp->edev; |
121 | |
122 | if (edev->ulp_tbl->msix_requested) |
123 | return edev->ulp_tbl->msix_base; |
124 | } |
125 | return 0; |
126 | } |
127 | |
128 | int bnxt_get_ulp_stat_ctxs(struct bnxt *bp) |
129 | { |
130 | if (bnxt_ulp_registered(edev: bp->edev)) { |
131 | struct bnxt_en_dev *edev = bp->edev; |
132 | |
133 | if (edev->ulp_tbl->msix_requested) |
134 | return BNXT_MIN_ROCE_STAT_CTXS; |
135 | } |
136 | |
137 | return 0; |
138 | } |
139 | |
140 | int bnxt_send_msg(struct bnxt_en_dev *edev, |
141 | struct bnxt_fw_msg *fw_msg) |
142 | { |
143 | struct net_device *dev = edev->net; |
144 | struct bnxt *bp = netdev_priv(dev); |
145 | struct output *resp; |
146 | struct input *req; |
147 | u32 resp_len; |
148 | int rc; |
149 | |
150 | if (bp->fw_reset_state) |
151 | return -EBUSY; |
152 | |
153 | rc = hwrm_req_init(bp, req, 0 /* don't care */); |
154 | if (rc) |
155 | return rc; |
156 | |
157 | rc = hwrm_req_replace(bp, req, new_req: fw_msg->msg, len: fw_msg->msg_len); |
158 | if (rc) |
159 | return rc; |
160 | |
161 | hwrm_req_timeout(bp, req, timeout: fw_msg->timeout); |
162 | resp = hwrm_req_hold(bp, req); |
163 | rc = hwrm_req_send(bp, req); |
164 | resp_len = le16_to_cpu(resp->resp_len); |
165 | if (resp_len) { |
166 | if (fw_msg->resp_max_len < resp_len) |
167 | resp_len = fw_msg->resp_max_len; |
168 | |
169 | memcpy(fw_msg->resp, resp, resp_len); |
170 | } |
171 | hwrm_req_drop(bp, req); |
172 | return rc; |
173 | } |
174 | EXPORT_SYMBOL(bnxt_send_msg); |
175 | |
176 | void bnxt_ulp_stop(struct bnxt *bp) |
177 | { |
178 | struct bnxt_aux_priv *aux_priv = bp->aux_priv; |
179 | struct bnxt_en_dev *edev = bp->edev; |
180 | |
181 | if (!edev) |
182 | return; |
183 | |
184 | edev->flags |= BNXT_EN_FLAG_ULP_STOPPED; |
185 | if (aux_priv) { |
186 | struct auxiliary_device *adev; |
187 | |
188 | adev = &aux_priv->aux_dev; |
189 | if (adev->dev.driver) { |
190 | struct auxiliary_driver *adrv; |
191 | pm_message_t pm = {}; |
192 | |
193 | adrv = to_auxiliary_drv(drv: adev->dev.driver); |
194 | edev->en_state = bp->state; |
195 | adrv->suspend(adev, pm); |
196 | } |
197 | } |
198 | } |
199 | |
200 | void bnxt_ulp_start(struct bnxt *bp, int err) |
201 | { |
202 | struct bnxt_aux_priv *aux_priv = bp->aux_priv; |
203 | struct bnxt_en_dev *edev = bp->edev; |
204 | |
205 | if (!edev) |
206 | return; |
207 | |
208 | edev->flags &= ~BNXT_EN_FLAG_ULP_STOPPED; |
209 | |
210 | if (err) |
211 | return; |
212 | |
213 | if (edev->ulp_tbl->msix_requested) |
214 | bnxt_fill_msix_vecs(bp, ent: edev->msix_entries); |
215 | |
216 | if (aux_priv) { |
217 | struct auxiliary_device *adev; |
218 | |
219 | adev = &aux_priv->aux_dev; |
220 | if (adev->dev.driver) { |
221 | struct auxiliary_driver *adrv; |
222 | |
223 | adrv = to_auxiliary_drv(drv: adev->dev.driver); |
224 | edev->en_state = bp->state; |
225 | adrv->resume(adev); |
226 | } |
227 | } |
228 | |
229 | } |
230 | |
231 | void bnxt_ulp_irq_stop(struct bnxt *bp) |
232 | { |
233 | struct bnxt_en_dev *edev = bp->edev; |
234 | struct bnxt_ulp_ops *ops; |
235 | |
236 | if (!edev || !(edev->flags & BNXT_EN_FLAG_MSIX_REQUESTED)) |
237 | return; |
238 | |
239 | if (bnxt_ulp_registered(edev: bp->edev)) { |
240 | struct bnxt_ulp *ulp = edev->ulp_tbl; |
241 | |
242 | if (!ulp->msix_requested) |
243 | return; |
244 | |
245 | ops = rtnl_dereference(ulp->ulp_ops); |
246 | if (!ops || !ops->ulp_irq_stop) |
247 | return; |
248 | ops->ulp_irq_stop(ulp->handle); |
249 | } |
250 | } |
251 | |
252 | void bnxt_ulp_irq_restart(struct bnxt *bp, int err) |
253 | { |
254 | struct bnxt_en_dev *edev = bp->edev; |
255 | struct bnxt_ulp_ops *ops; |
256 | |
257 | if (!edev || !(edev->flags & BNXT_EN_FLAG_MSIX_REQUESTED)) |
258 | return; |
259 | |
260 | if (bnxt_ulp_registered(edev: bp->edev)) { |
261 | struct bnxt_ulp *ulp = edev->ulp_tbl; |
262 | struct bnxt_msix_entry *ent = NULL; |
263 | |
264 | if (!ulp->msix_requested) |
265 | return; |
266 | |
267 | ops = rtnl_dereference(ulp->ulp_ops); |
268 | if (!ops || !ops->ulp_irq_restart) |
269 | return; |
270 | |
271 | if (!err) { |
272 | ent = kcalloc(n: ulp->msix_requested, size: sizeof(*ent), |
273 | GFP_KERNEL); |
274 | if (!ent) |
275 | return; |
276 | bnxt_fill_msix_vecs(bp, ent); |
277 | } |
278 | ops->ulp_irq_restart(ulp->handle, ent); |
279 | kfree(objp: ent); |
280 | } |
281 | } |
282 | |
283 | int bnxt_register_async_events(struct bnxt_en_dev *edev, |
284 | unsigned long *events_bmap, |
285 | u16 max_id) |
286 | { |
287 | struct net_device *dev = edev->net; |
288 | struct bnxt *bp = netdev_priv(dev); |
289 | struct bnxt_ulp *ulp; |
290 | |
291 | ulp = edev->ulp_tbl; |
292 | ulp->async_events_bmap = events_bmap; |
293 | /* Make sure bnxt_ulp_async_events() sees this order */ |
294 | smp_wmb(); |
295 | ulp->max_async_event_id = max_id; |
296 | bnxt_hwrm_func_drv_rgtr(bp, bmap: events_bmap, bmap_size: max_id + 1, async_only: true); |
297 | return 0; |
298 | } |
299 | EXPORT_SYMBOL(bnxt_register_async_events); |
300 | |
301 | void bnxt_rdma_aux_device_uninit(struct bnxt *bp) |
302 | { |
303 | struct bnxt_aux_priv *aux_priv; |
304 | struct auxiliary_device *adev; |
305 | |
306 | /* Skip if no auxiliary device init was done. */ |
307 | if (!bp->aux_priv) |
308 | return; |
309 | |
310 | aux_priv = bp->aux_priv; |
311 | adev = &aux_priv->aux_dev; |
312 | auxiliary_device_delete(auxdev: adev); |
313 | auxiliary_device_uninit(auxdev: adev); |
314 | } |
315 | |
316 | static void bnxt_aux_dev_release(struct device *dev) |
317 | { |
318 | struct bnxt_aux_priv *aux_priv = |
319 | container_of(dev, struct bnxt_aux_priv, aux_dev.dev); |
320 | struct bnxt *bp = netdev_priv(dev: aux_priv->edev->net); |
321 | |
322 | ida_free(&bnxt_aux_dev_ids, id: aux_priv->id); |
323 | kfree(objp: aux_priv->edev->ulp_tbl); |
324 | bp->edev = NULL; |
325 | kfree(objp: aux_priv->edev); |
326 | kfree(objp: aux_priv); |
327 | bp->aux_priv = NULL; |
328 | } |
329 | |
330 | static void bnxt_set_edev_info(struct bnxt_en_dev *edev, struct bnxt *bp) |
331 | { |
332 | edev->net = bp->dev; |
333 | edev->pdev = bp->pdev; |
334 | edev->l2_db_size = bp->db_size; |
335 | edev->l2_db_size_nc = bp->db_size; |
336 | edev->l2_db_offset = bp->db_offset; |
337 | |
338 | if (bp->flags & BNXT_FLAG_ROCEV1_CAP) |
339 | edev->flags |= BNXT_EN_FLAG_ROCEV1_CAP; |
340 | if (bp->flags & BNXT_FLAG_ROCEV2_CAP) |
341 | edev->flags |= BNXT_EN_FLAG_ROCEV2_CAP; |
342 | if (bp->flags & BNXT_FLAG_VF) |
343 | edev->flags |= BNXT_EN_FLAG_VF; |
344 | |
345 | edev->chip_num = bp->chip_num; |
346 | edev->hw_ring_stats_size = bp->hw_ring_stats_size; |
347 | edev->pf_port_id = bp->pf.port_id; |
348 | edev->en_state = bp->state; |
349 | edev->bar0 = bp->bar0; |
350 | edev->ulp_tbl->msix_requested = bnxt_get_ulp_msix_num(bp); |
351 | } |
352 | |
353 | void bnxt_rdma_aux_device_init(struct bnxt *bp) |
354 | { |
355 | struct auxiliary_device *aux_dev; |
356 | struct bnxt_aux_priv *aux_priv; |
357 | struct bnxt_en_dev *edev; |
358 | struct bnxt_ulp *ulp; |
359 | int rc; |
360 | |
361 | if (!(bp->flags & BNXT_FLAG_ROCE_CAP)) |
362 | return; |
363 | |
364 | aux_priv = kzalloc(size: sizeof(*bp->aux_priv), GFP_KERNEL); |
365 | if (!aux_priv) |
366 | goto exit; |
367 | |
368 | aux_priv->id = ida_alloc(ida: &bnxt_aux_dev_ids, GFP_KERNEL); |
369 | if (aux_priv->id < 0) { |
370 | netdev_warn(dev: bp->dev, |
371 | format: "ida alloc failed for ROCE auxiliary device\n" ); |
372 | kfree(objp: aux_priv); |
373 | goto exit; |
374 | } |
375 | |
376 | aux_dev = &aux_priv->aux_dev; |
377 | aux_dev->id = aux_priv->id; |
378 | aux_dev->name = "rdma" ; |
379 | aux_dev->dev.parent = &bp->pdev->dev; |
380 | aux_dev->dev.release = bnxt_aux_dev_release; |
381 | |
382 | rc = auxiliary_device_init(auxdev: aux_dev); |
383 | if (rc) { |
384 | ida_free(&bnxt_aux_dev_ids, id: aux_priv->id); |
385 | kfree(objp: aux_priv); |
386 | goto exit; |
387 | } |
388 | bp->aux_priv = aux_priv; |
389 | |
390 | /* From this point, all cleanup will happen via the .release callback & |
391 | * any error unwinding will need to include a call to |
392 | * auxiliary_device_uninit. |
393 | */ |
394 | edev = kzalloc(size: sizeof(*edev), GFP_KERNEL); |
395 | if (!edev) |
396 | goto aux_dev_uninit; |
397 | |
398 | aux_priv->edev = edev; |
399 | |
400 | ulp = kzalloc(size: sizeof(*ulp), GFP_KERNEL); |
401 | if (!ulp) |
402 | goto aux_dev_uninit; |
403 | |
404 | edev->ulp_tbl = ulp; |
405 | bp->edev = edev; |
406 | bnxt_set_edev_info(edev, bp); |
407 | |
408 | rc = auxiliary_device_add(aux_dev); |
409 | if (rc) { |
410 | netdev_warn(dev: bp->dev, |
411 | format: "Failed to add auxiliary device for ROCE\n" ); |
412 | goto aux_dev_uninit; |
413 | } |
414 | |
415 | return; |
416 | |
417 | aux_dev_uninit: |
418 | auxiliary_device_uninit(auxdev: aux_dev); |
419 | exit: |
420 | bp->flags &= ~BNXT_FLAG_ROCE_CAP; |
421 | } |
422 | |