1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Marvell RVU Physical Function ethernet driver |
3 | * |
4 | * Copyright (C) 2020 Marvell. |
5 | * |
6 | */ |
7 | |
8 | #include <linux/module.h> |
9 | #include <linux/interrupt.h> |
10 | #include <linux/pci.h> |
11 | #include <linux/etherdevice.h> |
12 | #include <linux/of.h> |
13 | #include <linux/if_vlan.h> |
14 | #include <linux/iommu.h> |
15 | #include <net/ip.h> |
16 | #include <linux/bpf.h> |
17 | #include <linux/bpf_trace.h> |
18 | #include <linux/bitfield.h> |
19 | #include <net/page_pool/types.h> |
20 | |
21 | #include "otx2_reg.h" |
22 | #include "otx2_common.h" |
23 | #include "otx2_txrx.h" |
24 | #include "otx2_struct.h" |
25 | #include "otx2_ptp.h" |
26 | #include "cn10k.h" |
27 | #include "qos.h" |
28 | #include <rvu_trace.h> |
29 | |
30 | #define DRV_NAME "rvu_nicpf" |
31 | #define DRV_STRING "Marvell RVU NIC Physical Function Driver" |
32 | |
33 | /* Supported devices */ |
34 | static const struct pci_device_id otx2_pf_id_table[] = { |
35 | { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_RVU_PF) }, |
36 | { 0, } /* end of table */ |
37 | }; |
38 | |
39 | MODULE_AUTHOR("Sunil Goutham <sgoutham@marvell.com>" ); |
40 | MODULE_DESCRIPTION(DRV_STRING); |
41 | MODULE_LICENSE("GPL v2" ); |
42 | MODULE_DEVICE_TABLE(pci, otx2_pf_id_table); |
43 | |
44 | static void otx2_vf_link_event_task(struct work_struct *work); |
45 | |
46 | enum { |
47 | TYPE_PFAF, |
48 | TYPE_PFVF, |
49 | }; |
50 | |
51 | static int otx2_config_hw_tx_tstamp(struct otx2_nic *pfvf, bool enable); |
52 | static int otx2_config_hw_rx_tstamp(struct otx2_nic *pfvf, bool enable); |
53 | |
54 | static int otx2_change_mtu(struct net_device *netdev, int new_mtu) |
55 | { |
56 | struct otx2_nic *pf = netdev_priv(dev: netdev); |
57 | bool if_up = netif_running(dev: netdev); |
58 | int err = 0; |
59 | |
60 | if (pf->xdp_prog && new_mtu > MAX_XDP_MTU) { |
61 | netdev_warn(dev: netdev, format: "Jumbo frames not yet supported with XDP, current MTU %d.\n" , |
62 | netdev->mtu); |
63 | return -EINVAL; |
64 | } |
65 | if (if_up) |
66 | otx2_stop(netdev); |
67 | |
68 | netdev_info(dev: netdev, format: "Changing MTU from %d to %d\n" , |
69 | netdev->mtu, new_mtu); |
70 | netdev->mtu = new_mtu; |
71 | |
72 | if (if_up) |
73 | err = otx2_open(netdev); |
74 | |
75 | return err; |
76 | } |
77 | |
78 | static void otx2_disable_flr_me_intr(struct otx2_nic *pf) |
79 | { |
80 | int irq, vfs = pf->total_vfs; |
81 | |
82 | /* Disable VFs ME interrupts */ |
83 | otx2_write64(nic: pf, RVU_PF_VFME_INT_ENA_W1CX(0), INTR_MASK(vfs)); |
84 | irq = pci_irq_vector(dev: pf->pdev, nr: RVU_PF_INT_VEC_VFME0); |
85 | free_irq(irq, pf); |
86 | |
87 | /* Disable VFs FLR interrupts */ |
88 | otx2_write64(nic: pf, RVU_PF_VFFLR_INT_ENA_W1CX(0), INTR_MASK(vfs)); |
89 | irq = pci_irq_vector(dev: pf->pdev, nr: RVU_PF_INT_VEC_VFFLR0); |
90 | free_irq(irq, pf); |
91 | |
92 | if (vfs <= 64) |
93 | return; |
94 | |
95 | otx2_write64(nic: pf, RVU_PF_VFME_INT_ENA_W1CX(1), INTR_MASK(vfs - 64)); |
96 | irq = pci_irq_vector(dev: pf->pdev, nr: RVU_PF_INT_VEC_VFME1); |
97 | free_irq(irq, pf); |
98 | |
99 | otx2_write64(nic: pf, RVU_PF_VFFLR_INT_ENA_W1CX(1), INTR_MASK(vfs - 64)); |
100 | irq = pci_irq_vector(dev: pf->pdev, nr: RVU_PF_INT_VEC_VFFLR1); |
101 | free_irq(irq, pf); |
102 | } |
103 | |
104 | static void otx2_flr_wq_destroy(struct otx2_nic *pf) |
105 | { |
106 | if (!pf->flr_wq) |
107 | return; |
108 | destroy_workqueue(wq: pf->flr_wq); |
109 | pf->flr_wq = NULL; |
110 | devm_kfree(dev: pf->dev, p: pf->flr_wrk); |
111 | } |
112 | |
113 | static void otx2_flr_handler(struct work_struct *work) |
114 | { |
115 | struct flr_work *flrwork = container_of(work, struct flr_work, work); |
116 | struct otx2_nic *pf = flrwork->pf; |
117 | struct mbox *mbox = &pf->mbox; |
118 | struct msg_req *req; |
119 | int vf, reg = 0; |
120 | |
121 | vf = flrwork - pf->flr_wrk; |
122 | |
123 | mutex_lock(&mbox->lock); |
124 | req = otx2_mbox_alloc_msg_vf_flr(mbox); |
125 | if (!req) { |
126 | mutex_unlock(lock: &mbox->lock); |
127 | return; |
128 | } |
129 | req->hdr.pcifunc &= RVU_PFVF_FUNC_MASK; |
130 | req->hdr.pcifunc |= (vf + 1) & RVU_PFVF_FUNC_MASK; |
131 | |
132 | if (!otx2_sync_mbox_msg(mbox: &pf->mbox)) { |
133 | if (vf >= 64) { |
134 | reg = 1; |
135 | vf = vf - 64; |
136 | } |
137 | /* clear transcation pending bit */ |
138 | otx2_write64(nic: pf, RVU_PF_VFTRPENDX(reg), BIT_ULL(vf)); |
139 | otx2_write64(nic: pf, RVU_PF_VFFLR_INT_ENA_W1SX(reg), BIT_ULL(vf)); |
140 | } |
141 | |
142 | mutex_unlock(lock: &mbox->lock); |
143 | } |
144 | |
145 | static irqreturn_t otx2_pf_flr_intr_handler(int irq, void *pf_irq) |
146 | { |
147 | struct otx2_nic *pf = (struct otx2_nic *)pf_irq; |
148 | int reg, dev, vf, start_vf, num_reg = 1; |
149 | u64 intr; |
150 | |
151 | if (pf->total_vfs > 64) |
152 | num_reg = 2; |
153 | |
154 | for (reg = 0; reg < num_reg; reg++) { |
155 | intr = otx2_read64(nic: pf, RVU_PF_VFFLR_INTX(reg)); |
156 | if (!intr) |
157 | continue; |
158 | start_vf = 64 * reg; |
159 | for (vf = 0; vf < 64; vf++) { |
160 | if (!(intr & BIT_ULL(vf))) |
161 | continue; |
162 | dev = vf + start_vf; |
163 | queue_work(wq: pf->flr_wq, work: &pf->flr_wrk[dev].work); |
164 | /* Clear interrupt */ |
165 | otx2_write64(nic: pf, RVU_PF_VFFLR_INTX(reg), BIT_ULL(vf)); |
166 | /* Disable the interrupt */ |
167 | otx2_write64(nic: pf, RVU_PF_VFFLR_INT_ENA_W1CX(reg), |
168 | BIT_ULL(vf)); |
169 | } |
170 | } |
171 | return IRQ_HANDLED; |
172 | } |
173 | |
174 | static irqreturn_t otx2_pf_me_intr_handler(int irq, void *pf_irq) |
175 | { |
176 | struct otx2_nic *pf = (struct otx2_nic *)pf_irq; |
177 | int vf, reg, num_reg = 1; |
178 | u64 intr; |
179 | |
180 | if (pf->total_vfs > 64) |
181 | num_reg = 2; |
182 | |
183 | for (reg = 0; reg < num_reg; reg++) { |
184 | intr = otx2_read64(nic: pf, RVU_PF_VFME_INTX(reg)); |
185 | if (!intr) |
186 | continue; |
187 | for (vf = 0; vf < 64; vf++) { |
188 | if (!(intr & BIT_ULL(vf))) |
189 | continue; |
190 | /* clear trpend bit */ |
191 | otx2_write64(nic: pf, RVU_PF_VFTRPENDX(reg), BIT_ULL(vf)); |
192 | /* clear interrupt */ |
193 | otx2_write64(nic: pf, RVU_PF_VFME_INTX(reg), BIT_ULL(vf)); |
194 | } |
195 | } |
196 | return IRQ_HANDLED; |
197 | } |
198 | |
199 | static int otx2_register_flr_me_intr(struct otx2_nic *pf, int numvfs) |
200 | { |
201 | struct otx2_hw *hw = &pf->hw; |
202 | char *irq_name; |
203 | int ret; |
204 | |
205 | /* Register ME interrupt handler*/ |
206 | irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFME0 * NAME_SIZE]; |
207 | snprintf(buf: irq_name, NAME_SIZE, fmt: "RVUPF%d_ME0" , rvu_get_pf(pcifunc: pf->pcifunc)); |
208 | ret = request_irq(irq: pci_irq_vector(dev: pf->pdev, nr: RVU_PF_INT_VEC_VFME0), |
209 | handler: otx2_pf_me_intr_handler, flags: 0, name: irq_name, dev: pf); |
210 | if (ret) { |
211 | dev_err(pf->dev, |
212 | "RVUPF: IRQ registration failed for ME0\n" ); |
213 | } |
214 | |
215 | /* Register FLR interrupt handler */ |
216 | irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFFLR0 * NAME_SIZE]; |
217 | snprintf(buf: irq_name, NAME_SIZE, fmt: "RVUPF%d_FLR0" , rvu_get_pf(pcifunc: pf->pcifunc)); |
218 | ret = request_irq(irq: pci_irq_vector(dev: pf->pdev, nr: RVU_PF_INT_VEC_VFFLR0), |
219 | handler: otx2_pf_flr_intr_handler, flags: 0, name: irq_name, dev: pf); |
220 | if (ret) { |
221 | dev_err(pf->dev, |
222 | "RVUPF: IRQ registration failed for FLR0\n" ); |
223 | return ret; |
224 | } |
225 | |
226 | if (numvfs > 64) { |
227 | irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFME1 * NAME_SIZE]; |
228 | snprintf(buf: irq_name, NAME_SIZE, fmt: "RVUPF%d_ME1" , |
229 | rvu_get_pf(pcifunc: pf->pcifunc)); |
230 | ret = request_irq(irq: pci_irq_vector |
231 | (dev: pf->pdev, nr: RVU_PF_INT_VEC_VFME1), |
232 | handler: otx2_pf_me_intr_handler, flags: 0, name: irq_name, dev: pf); |
233 | if (ret) { |
234 | dev_err(pf->dev, |
235 | "RVUPF: IRQ registration failed for ME1\n" ); |
236 | } |
237 | irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFFLR1 * NAME_SIZE]; |
238 | snprintf(buf: irq_name, NAME_SIZE, fmt: "RVUPF%d_FLR1" , |
239 | rvu_get_pf(pcifunc: pf->pcifunc)); |
240 | ret = request_irq(irq: pci_irq_vector |
241 | (dev: pf->pdev, nr: RVU_PF_INT_VEC_VFFLR1), |
242 | handler: otx2_pf_flr_intr_handler, flags: 0, name: irq_name, dev: pf); |
243 | if (ret) { |
244 | dev_err(pf->dev, |
245 | "RVUPF: IRQ registration failed for FLR1\n" ); |
246 | return ret; |
247 | } |
248 | } |
249 | |
250 | /* Enable ME interrupt for all VFs*/ |
251 | otx2_write64(nic: pf, RVU_PF_VFME_INTX(0), INTR_MASK(numvfs)); |
252 | otx2_write64(nic: pf, RVU_PF_VFME_INT_ENA_W1SX(0), INTR_MASK(numvfs)); |
253 | |
254 | /* Enable FLR interrupt for all VFs*/ |
255 | otx2_write64(nic: pf, RVU_PF_VFFLR_INTX(0), INTR_MASK(numvfs)); |
256 | otx2_write64(nic: pf, RVU_PF_VFFLR_INT_ENA_W1SX(0), INTR_MASK(numvfs)); |
257 | |
258 | if (numvfs > 64) { |
259 | numvfs -= 64; |
260 | |
261 | otx2_write64(nic: pf, RVU_PF_VFME_INTX(1), INTR_MASK(numvfs)); |
262 | otx2_write64(nic: pf, RVU_PF_VFME_INT_ENA_W1SX(1), |
263 | INTR_MASK(numvfs)); |
264 | |
265 | otx2_write64(nic: pf, RVU_PF_VFFLR_INTX(1), INTR_MASK(numvfs)); |
266 | otx2_write64(nic: pf, RVU_PF_VFFLR_INT_ENA_W1SX(1), |
267 | INTR_MASK(numvfs)); |
268 | } |
269 | return 0; |
270 | } |
271 | |
272 | static int otx2_pf_flr_init(struct otx2_nic *pf, int num_vfs) |
273 | { |
274 | int vf; |
275 | |
276 | pf->flr_wq = alloc_ordered_workqueue("otx2_pf_flr_wq" , WQ_HIGHPRI); |
277 | if (!pf->flr_wq) |
278 | return -ENOMEM; |
279 | |
280 | pf->flr_wrk = devm_kcalloc(dev: pf->dev, n: num_vfs, |
281 | size: sizeof(struct flr_work), GFP_KERNEL); |
282 | if (!pf->flr_wrk) { |
283 | destroy_workqueue(wq: pf->flr_wq); |
284 | return -ENOMEM; |
285 | } |
286 | |
287 | for (vf = 0; vf < num_vfs; vf++) { |
288 | pf->flr_wrk[vf].pf = pf; |
289 | INIT_WORK(&pf->flr_wrk[vf].work, otx2_flr_handler); |
290 | } |
291 | |
292 | return 0; |
293 | } |
294 | |
295 | static void otx2_queue_vf_work(struct mbox *mw, struct workqueue_struct *mbox_wq, |
296 | int first, int mdevs, u64 intr) |
297 | { |
298 | struct otx2_mbox_dev *mdev; |
299 | struct otx2_mbox *mbox; |
300 | struct mbox_hdr *hdr; |
301 | int i; |
302 | |
303 | for (i = first; i < mdevs; i++) { |
304 | /* start from 0 */ |
305 | if (!(intr & BIT_ULL(i - first))) |
306 | continue; |
307 | |
308 | mbox = &mw->mbox; |
309 | mdev = &mbox->dev[i]; |
310 | hdr = mdev->mbase + mbox->rx_start; |
311 | /* The hdr->num_msgs is set to zero immediately in the interrupt |
312 | * handler to ensure that it holds a correct value next time |
313 | * when the interrupt handler is called. pf->mw[i].num_msgs |
314 | * holds the data for use in otx2_pfvf_mbox_handler and |
315 | * pf->mw[i].up_num_msgs holds the data for use in |
316 | * otx2_pfvf_mbox_up_handler. |
317 | */ |
318 | if (hdr->num_msgs) { |
319 | mw[i].num_msgs = hdr->num_msgs; |
320 | hdr->num_msgs = 0; |
321 | queue_work(wq: mbox_wq, work: &mw[i].mbox_wrk); |
322 | } |
323 | |
324 | mbox = &mw->mbox_up; |
325 | mdev = &mbox->dev[i]; |
326 | hdr = mdev->mbase + mbox->rx_start; |
327 | if (hdr->num_msgs) { |
328 | mw[i].up_num_msgs = hdr->num_msgs; |
329 | hdr->num_msgs = 0; |
330 | queue_work(wq: mbox_wq, work: &mw[i].mbox_up_wrk); |
331 | } |
332 | } |
333 | } |
334 | |
335 | static void otx2_forward_msg_pfvf(struct otx2_mbox_dev *mdev, |
336 | struct otx2_mbox *pfvf_mbox, void *bbuf_base, |
337 | int devid) |
338 | { |
339 | struct otx2_mbox_dev *src_mdev = mdev; |
340 | int offset; |
341 | |
342 | /* Msgs are already copied, trigger VF's mbox irq */ |
343 | smp_wmb(); |
344 | |
345 | otx2_mbox_wait_for_zero(mbox: pfvf_mbox, devid); |
346 | |
347 | offset = pfvf_mbox->trigger | (devid << pfvf_mbox->tr_shift); |
348 | writeq(MBOX_DOWN_MSG, addr: (void __iomem *)pfvf_mbox->reg_base + offset); |
349 | |
350 | /* Restore VF's mbox bounce buffer region address */ |
351 | src_mdev->mbase = bbuf_base; |
352 | } |
353 | |
354 | static int otx2_forward_vf_mbox_msgs(struct otx2_nic *pf, |
355 | struct otx2_mbox *src_mbox, |
356 | int dir, int vf, int num_msgs) |
357 | { |
358 | struct otx2_mbox_dev *src_mdev, *dst_mdev; |
359 | struct mbox_hdr *mbox_hdr; |
360 | struct mbox_hdr *req_hdr; |
361 | struct mbox *dst_mbox; |
362 | int dst_size, err; |
363 | |
364 | if (dir == MBOX_DIR_PFAF) { |
365 | /* Set VF's mailbox memory as PF's bounce buffer memory, so |
366 | * that explicit copying of VF's msgs to PF=>AF mbox region |
367 | * and AF=>PF responses to VF's mbox region can be avoided. |
368 | */ |
369 | src_mdev = &src_mbox->dev[vf]; |
370 | mbox_hdr = src_mbox->hwbase + |
371 | src_mbox->rx_start + (vf * MBOX_SIZE); |
372 | |
373 | dst_mbox = &pf->mbox; |
374 | dst_size = dst_mbox->mbox.tx_size - |
375 | ALIGN(sizeof(*mbox_hdr), MBOX_MSG_ALIGN); |
376 | /* Check if msgs fit into destination area and has valid size */ |
377 | if (mbox_hdr->msg_size > dst_size || !mbox_hdr->msg_size) |
378 | return -EINVAL; |
379 | |
380 | dst_mdev = &dst_mbox->mbox.dev[0]; |
381 | |
382 | mutex_lock(&pf->mbox.lock); |
383 | dst_mdev->mbase = src_mdev->mbase; |
384 | dst_mdev->msg_size = mbox_hdr->msg_size; |
385 | dst_mdev->num_msgs = num_msgs; |
386 | err = otx2_sync_mbox_msg(mbox: dst_mbox); |
387 | /* Error code -EIO indicate there is a communication failure |
388 | * to the AF. Rest of the error codes indicate that AF processed |
389 | * VF messages and set the error codes in response messages |
390 | * (if any) so simply forward responses to VF. |
391 | */ |
392 | if (err == -EIO) { |
393 | dev_warn(pf->dev, |
394 | "AF not responding to VF%d messages\n" , vf); |
395 | /* restore PF mbase and exit */ |
396 | dst_mdev->mbase = pf->mbox.bbuf_base; |
397 | mutex_unlock(lock: &pf->mbox.lock); |
398 | return err; |
399 | } |
400 | /* At this point, all the VF messages sent to AF are acked |
401 | * with proper responses and responses are copied to VF |
402 | * mailbox hence raise interrupt to VF. |
403 | */ |
404 | req_hdr = (struct mbox_hdr *)(dst_mdev->mbase + |
405 | dst_mbox->mbox.rx_start); |
406 | req_hdr->num_msgs = num_msgs; |
407 | |
408 | otx2_forward_msg_pfvf(mdev: dst_mdev, pfvf_mbox: &pf->mbox_pfvf[0].mbox, |
409 | bbuf_base: pf->mbox.bbuf_base, devid: vf); |
410 | mutex_unlock(lock: &pf->mbox.lock); |
411 | } else if (dir == MBOX_DIR_PFVF_UP) { |
412 | src_mdev = &src_mbox->dev[0]; |
413 | mbox_hdr = src_mbox->hwbase + src_mbox->rx_start; |
414 | req_hdr = (struct mbox_hdr *)(src_mdev->mbase + |
415 | src_mbox->rx_start); |
416 | req_hdr->num_msgs = num_msgs; |
417 | |
418 | dst_mbox = &pf->mbox_pfvf[0]; |
419 | dst_size = dst_mbox->mbox_up.tx_size - |
420 | ALIGN(sizeof(*mbox_hdr), MBOX_MSG_ALIGN); |
421 | /* Check if msgs fit into destination area */ |
422 | if (mbox_hdr->msg_size > dst_size) |
423 | return -EINVAL; |
424 | |
425 | dst_mdev = &dst_mbox->mbox_up.dev[vf]; |
426 | dst_mdev->mbase = src_mdev->mbase; |
427 | dst_mdev->msg_size = mbox_hdr->msg_size; |
428 | dst_mdev->num_msgs = mbox_hdr->num_msgs; |
429 | err = otx2_sync_mbox_up_msg(mbox: dst_mbox, devid: vf); |
430 | if (err) { |
431 | dev_warn(pf->dev, |
432 | "VF%d is not responding to mailbox\n" , vf); |
433 | return err; |
434 | } |
435 | } else if (dir == MBOX_DIR_VFPF_UP) { |
436 | req_hdr = (struct mbox_hdr *)(src_mbox->dev[0].mbase + |
437 | src_mbox->rx_start); |
438 | req_hdr->num_msgs = num_msgs; |
439 | otx2_forward_msg_pfvf(mdev: &pf->mbox_pfvf->mbox_up.dev[vf], |
440 | pfvf_mbox: &pf->mbox.mbox_up, |
441 | bbuf_base: pf->mbox_pfvf[vf].bbuf_base, |
442 | devid: 0); |
443 | } |
444 | |
445 | return 0; |
446 | } |
447 | |
448 | static void otx2_pfvf_mbox_handler(struct work_struct *work) |
449 | { |
450 | struct mbox_msghdr *msg = NULL; |
451 | int offset, vf_idx, id, err; |
452 | struct otx2_mbox_dev *mdev; |
453 | struct mbox_hdr *req_hdr; |
454 | struct otx2_mbox *mbox; |
455 | struct mbox *vf_mbox; |
456 | struct otx2_nic *pf; |
457 | |
458 | vf_mbox = container_of(work, struct mbox, mbox_wrk); |
459 | pf = vf_mbox->pfvf; |
460 | vf_idx = vf_mbox - pf->mbox_pfvf; |
461 | |
462 | mbox = &pf->mbox_pfvf[0].mbox; |
463 | mdev = &mbox->dev[vf_idx]; |
464 | req_hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start); |
465 | |
466 | offset = ALIGN(sizeof(*req_hdr), MBOX_MSG_ALIGN); |
467 | |
468 | for (id = 0; id < vf_mbox->num_msgs; id++) { |
469 | msg = (struct mbox_msghdr *)(mdev->mbase + mbox->rx_start + |
470 | offset); |
471 | |
472 | if (msg->sig != OTX2_MBOX_REQ_SIG) |
473 | goto inval_msg; |
474 | |
475 | /* Set VF's number in each of the msg */ |
476 | msg->pcifunc &= RVU_PFVF_FUNC_MASK; |
477 | msg->pcifunc |= (vf_idx + 1) & RVU_PFVF_FUNC_MASK; |
478 | offset = msg->next_msgoff; |
479 | } |
480 | err = otx2_forward_vf_mbox_msgs(pf, src_mbox: mbox, MBOX_DIR_PFAF, vf: vf_idx, |
481 | num_msgs: vf_mbox->num_msgs); |
482 | if (err) |
483 | goto inval_msg; |
484 | return; |
485 | |
486 | inval_msg: |
487 | otx2_reply_invalid_msg(mbox, devid: vf_idx, pcifunc: 0, id: msg->id); |
488 | otx2_mbox_msg_send(mbox, devid: vf_idx); |
489 | } |
490 | |
491 | static void otx2_pfvf_mbox_up_handler(struct work_struct *work) |
492 | { |
493 | struct mbox *vf_mbox = container_of(work, struct mbox, mbox_up_wrk); |
494 | struct otx2_nic *pf = vf_mbox->pfvf; |
495 | struct otx2_mbox_dev *mdev; |
496 | int offset, id, vf_idx = 0; |
497 | struct mbox_hdr *rsp_hdr; |
498 | struct mbox_msghdr *msg; |
499 | struct otx2_mbox *mbox; |
500 | |
501 | vf_idx = vf_mbox - pf->mbox_pfvf; |
502 | mbox = &pf->mbox_pfvf[0].mbox_up; |
503 | mdev = &mbox->dev[vf_idx]; |
504 | |
505 | rsp_hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start); |
506 | offset = mbox->rx_start + ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN); |
507 | |
508 | for (id = 0; id < vf_mbox->up_num_msgs; id++) { |
509 | msg = mdev->mbase + offset; |
510 | |
511 | if (msg->id >= MBOX_MSG_MAX) { |
512 | dev_err(pf->dev, |
513 | "Mbox msg with unknown ID 0x%x\n" , msg->id); |
514 | goto end; |
515 | } |
516 | |
517 | if (msg->sig != OTX2_MBOX_RSP_SIG) { |
518 | dev_err(pf->dev, |
519 | "Mbox msg with wrong signature %x, ID 0x%x\n" , |
520 | msg->sig, msg->id); |
521 | goto end; |
522 | } |
523 | |
524 | switch (msg->id) { |
525 | case MBOX_MSG_CGX_LINK_EVENT: |
526 | break; |
527 | default: |
528 | if (msg->rc) |
529 | dev_err(pf->dev, |
530 | "Mbox msg response has err %d, ID 0x%x\n" , |
531 | msg->rc, msg->id); |
532 | break; |
533 | } |
534 | |
535 | end: |
536 | offset = mbox->rx_start + msg->next_msgoff; |
537 | if (mdev->msgs_acked == (vf_mbox->up_num_msgs - 1)) |
538 | __otx2_mbox_reset(mbox, devid: vf_idx); |
539 | mdev->msgs_acked++; |
540 | } |
541 | } |
542 | |
543 | static irqreturn_t otx2_pfvf_mbox_intr_handler(int irq, void *pf_irq) |
544 | { |
545 | struct otx2_nic *pf = (struct otx2_nic *)(pf_irq); |
546 | int vfs = pf->total_vfs; |
547 | struct mbox *mbox; |
548 | u64 intr; |
549 | |
550 | mbox = pf->mbox_pfvf; |
551 | /* Handle VF interrupts */ |
552 | if (vfs > 64) { |
553 | intr = otx2_read64(nic: pf, RVU_PF_VFPF_MBOX_INTX(1)); |
554 | otx2_write64(nic: pf, RVU_PF_VFPF_MBOX_INTX(1), val: intr); |
555 | otx2_queue_vf_work(mw: mbox, mbox_wq: pf->mbox_pfvf_wq, first: 64, mdevs: vfs, intr); |
556 | if (intr) |
557 | trace_otx2_msg_interrupt(pdev: mbox->mbox.pdev, msg: "VF(s) to PF" , intr); |
558 | vfs = 64; |
559 | } |
560 | |
561 | intr = otx2_read64(nic: pf, RVU_PF_VFPF_MBOX_INTX(0)); |
562 | otx2_write64(nic: pf, RVU_PF_VFPF_MBOX_INTX(0), val: intr); |
563 | |
564 | otx2_queue_vf_work(mw: mbox, mbox_wq: pf->mbox_pfvf_wq, first: 0, mdevs: vfs, intr); |
565 | |
566 | if (intr) |
567 | trace_otx2_msg_interrupt(pdev: mbox->mbox.pdev, msg: "VF(s) to PF" , intr); |
568 | |
569 | return IRQ_HANDLED; |
570 | } |
571 | |
572 | static int otx2_pfvf_mbox_init(struct otx2_nic *pf, int numvfs) |
573 | { |
574 | void __iomem *hwbase; |
575 | struct mbox *mbox; |
576 | int err, vf; |
577 | u64 base; |
578 | |
579 | if (!numvfs) |
580 | return -EINVAL; |
581 | |
582 | pf->mbox_pfvf = devm_kcalloc(dev: &pf->pdev->dev, n: numvfs, |
583 | size: sizeof(struct mbox), GFP_KERNEL); |
584 | if (!pf->mbox_pfvf) |
585 | return -ENOMEM; |
586 | |
587 | pf->mbox_pfvf_wq = alloc_workqueue(fmt: "otx2_pfvf_mailbox" , |
588 | flags: WQ_UNBOUND | WQ_HIGHPRI | |
589 | WQ_MEM_RECLAIM, max_active: 0); |
590 | if (!pf->mbox_pfvf_wq) |
591 | return -ENOMEM; |
592 | |
593 | /* On CN10K platform, PF <-> VF mailbox region follows after |
594 | * PF <-> AF mailbox region. |
595 | */ |
596 | if (test_bit(CN10K_MBOX, &pf->hw.cap_flag)) |
597 | base = pci_resource_start(pf->pdev, PCI_MBOX_BAR_NUM) + |
598 | MBOX_SIZE; |
599 | else |
600 | base = readq(addr: (void __iomem *)((u64)pf->reg_base + |
601 | RVU_PF_VF_BAR4_ADDR)); |
602 | |
603 | hwbase = ioremap_wc(offset: base, MBOX_SIZE * pf->total_vfs); |
604 | if (!hwbase) { |
605 | err = -ENOMEM; |
606 | goto free_wq; |
607 | } |
608 | |
609 | mbox = &pf->mbox_pfvf[0]; |
610 | err = otx2_mbox_init(mbox: &mbox->mbox, hwbase, pdev: pf->pdev, reg_base: pf->reg_base, |
611 | MBOX_DIR_PFVF, ndevs: numvfs); |
612 | if (err) |
613 | goto free_iomem; |
614 | |
615 | err = otx2_mbox_init(mbox: &mbox->mbox_up, hwbase, pdev: pf->pdev, reg_base: pf->reg_base, |
616 | MBOX_DIR_PFVF_UP, ndevs: numvfs); |
617 | if (err) |
618 | goto free_iomem; |
619 | |
620 | for (vf = 0; vf < numvfs; vf++) { |
621 | mbox->pfvf = pf; |
622 | INIT_WORK(&mbox->mbox_wrk, otx2_pfvf_mbox_handler); |
623 | INIT_WORK(&mbox->mbox_up_wrk, otx2_pfvf_mbox_up_handler); |
624 | mbox++; |
625 | } |
626 | |
627 | return 0; |
628 | |
629 | free_iomem: |
630 | if (hwbase) |
631 | iounmap(addr: hwbase); |
632 | free_wq: |
633 | destroy_workqueue(wq: pf->mbox_pfvf_wq); |
634 | return err; |
635 | } |
636 | |
637 | static void otx2_pfvf_mbox_destroy(struct otx2_nic *pf) |
638 | { |
639 | struct mbox *mbox = &pf->mbox_pfvf[0]; |
640 | |
641 | if (!mbox) |
642 | return; |
643 | |
644 | if (pf->mbox_pfvf_wq) { |
645 | destroy_workqueue(wq: pf->mbox_pfvf_wq); |
646 | pf->mbox_pfvf_wq = NULL; |
647 | } |
648 | |
649 | if (mbox->mbox.hwbase) |
650 | iounmap(addr: mbox->mbox.hwbase); |
651 | |
652 | otx2_mbox_destroy(mbox: &mbox->mbox); |
653 | } |
654 | |
655 | static void otx2_enable_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs) |
656 | { |
657 | /* Clear PF <=> VF mailbox IRQ */ |
658 | otx2_write64(nic: pf, RVU_PF_VFPF_MBOX_INTX(0), val: ~0ull); |
659 | otx2_write64(nic: pf, RVU_PF_VFPF_MBOX_INTX(1), val: ~0ull); |
660 | |
661 | /* Enable PF <=> VF mailbox IRQ */ |
662 | otx2_write64(nic: pf, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(0), INTR_MASK(numvfs)); |
663 | if (numvfs > 64) { |
664 | numvfs -= 64; |
665 | otx2_write64(nic: pf, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(1), |
666 | INTR_MASK(numvfs)); |
667 | } |
668 | } |
669 | |
670 | static void otx2_disable_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs) |
671 | { |
672 | int vector; |
673 | |
674 | /* Disable PF <=> VF mailbox IRQ */ |
675 | otx2_write64(nic: pf, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(0), val: ~0ull); |
676 | otx2_write64(nic: pf, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(1), val: ~0ull); |
677 | |
678 | otx2_write64(nic: pf, RVU_PF_VFPF_MBOX_INTX(0), val: ~0ull); |
679 | vector = pci_irq_vector(dev: pf->pdev, nr: RVU_PF_INT_VEC_VFPF_MBOX0); |
680 | free_irq(vector, pf); |
681 | |
682 | if (numvfs > 64) { |
683 | otx2_write64(nic: pf, RVU_PF_VFPF_MBOX_INTX(1), val: ~0ull); |
684 | vector = pci_irq_vector(dev: pf->pdev, nr: RVU_PF_INT_VEC_VFPF_MBOX1); |
685 | free_irq(vector, pf); |
686 | } |
687 | } |
688 | |
689 | static int otx2_register_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs) |
690 | { |
691 | struct otx2_hw *hw = &pf->hw; |
692 | char *irq_name; |
693 | int err; |
694 | |
695 | /* Register MBOX0 interrupt handler */ |
696 | irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFPF_MBOX0 * NAME_SIZE]; |
697 | if (pf->pcifunc) |
698 | snprintf(buf: irq_name, NAME_SIZE, |
699 | fmt: "RVUPF%d_VF Mbox0" , rvu_get_pf(pcifunc: pf->pcifunc)); |
700 | else |
701 | snprintf(buf: irq_name, NAME_SIZE, fmt: "RVUPF_VF Mbox0" ); |
702 | err = request_irq(irq: pci_irq_vector(dev: pf->pdev, nr: RVU_PF_INT_VEC_VFPF_MBOX0), |
703 | handler: otx2_pfvf_mbox_intr_handler, flags: 0, name: irq_name, dev: pf); |
704 | if (err) { |
705 | dev_err(pf->dev, |
706 | "RVUPF: IRQ registration failed for PFVF mbox0 irq\n" ); |
707 | return err; |
708 | } |
709 | |
710 | if (numvfs > 64) { |
711 | /* Register MBOX1 interrupt handler */ |
712 | irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFPF_MBOX1 * NAME_SIZE]; |
713 | if (pf->pcifunc) |
714 | snprintf(buf: irq_name, NAME_SIZE, |
715 | fmt: "RVUPF%d_VF Mbox1" , rvu_get_pf(pcifunc: pf->pcifunc)); |
716 | else |
717 | snprintf(buf: irq_name, NAME_SIZE, fmt: "RVUPF_VF Mbox1" ); |
718 | err = request_irq(irq: pci_irq_vector(dev: pf->pdev, |
719 | nr: RVU_PF_INT_VEC_VFPF_MBOX1), |
720 | handler: otx2_pfvf_mbox_intr_handler, |
721 | flags: 0, name: irq_name, dev: pf); |
722 | if (err) { |
723 | dev_err(pf->dev, |
724 | "RVUPF: IRQ registration failed for PFVF mbox1 irq\n" ); |
725 | return err; |
726 | } |
727 | } |
728 | |
729 | otx2_enable_pfvf_mbox_intr(pf, numvfs); |
730 | |
731 | return 0; |
732 | } |
733 | |
734 | static void otx2_process_pfaf_mbox_msg(struct otx2_nic *pf, |
735 | struct mbox_msghdr *msg) |
736 | { |
737 | int devid; |
738 | |
739 | if (msg->id >= MBOX_MSG_MAX) { |
740 | dev_err(pf->dev, |
741 | "Mbox msg with unknown ID 0x%x\n" , msg->id); |
742 | return; |
743 | } |
744 | |
745 | if (msg->sig != OTX2_MBOX_RSP_SIG) { |
746 | dev_err(pf->dev, |
747 | "Mbox msg with wrong signature %x, ID 0x%x\n" , |
748 | msg->sig, msg->id); |
749 | return; |
750 | } |
751 | |
752 | /* message response heading VF */ |
753 | devid = msg->pcifunc & RVU_PFVF_FUNC_MASK; |
754 | if (devid) { |
755 | struct otx2_vf_config *config = &pf->vf_configs[devid - 1]; |
756 | struct delayed_work *dwork; |
757 | |
758 | switch (msg->id) { |
759 | case MBOX_MSG_NIX_LF_START_RX: |
760 | config->intf_down = false; |
761 | dwork = &config->link_event_work; |
762 | schedule_delayed_work(dwork, delay: msecs_to_jiffies(m: 100)); |
763 | break; |
764 | case MBOX_MSG_NIX_LF_STOP_RX: |
765 | config->intf_down = true; |
766 | break; |
767 | } |
768 | |
769 | return; |
770 | } |
771 | |
772 | switch (msg->id) { |
773 | case MBOX_MSG_READY: |
774 | pf->pcifunc = msg->pcifunc; |
775 | break; |
776 | case MBOX_MSG_MSIX_OFFSET: |
777 | mbox_handler_msix_offset(pfvf: pf, rsp: (struct msix_offset_rsp *)msg); |
778 | break; |
779 | case MBOX_MSG_NPA_LF_ALLOC: |
780 | mbox_handler_npa_lf_alloc(pfvf: pf, rsp: (struct npa_lf_alloc_rsp *)msg); |
781 | break; |
782 | case MBOX_MSG_NIX_LF_ALLOC: |
783 | mbox_handler_nix_lf_alloc(pfvf: pf, rsp: (struct nix_lf_alloc_rsp *)msg); |
784 | break; |
785 | case MBOX_MSG_NIX_BP_ENABLE: |
786 | mbox_handler_nix_bp_enable(pfvf: pf, rsp: (struct nix_bp_cfg_rsp *)msg); |
787 | break; |
788 | case MBOX_MSG_CGX_STATS: |
789 | mbox_handler_cgx_stats(pfvf: pf, rsp: (struct cgx_stats_rsp *)msg); |
790 | break; |
791 | case MBOX_MSG_CGX_FEC_STATS: |
792 | mbox_handler_cgx_fec_stats(pfvf: pf, rsp: (struct cgx_fec_stats_rsp *)msg); |
793 | break; |
794 | default: |
795 | if (msg->rc) |
796 | dev_err(pf->dev, |
797 | "Mbox msg response has err %d, ID 0x%x\n" , |
798 | msg->rc, msg->id); |
799 | break; |
800 | } |
801 | } |
802 | |
803 | static void otx2_pfaf_mbox_handler(struct work_struct *work) |
804 | { |
805 | struct otx2_mbox_dev *mdev; |
806 | struct mbox_hdr *rsp_hdr; |
807 | struct mbox_msghdr *msg; |
808 | struct otx2_mbox *mbox; |
809 | struct mbox *af_mbox; |
810 | struct otx2_nic *pf; |
811 | int offset, id; |
812 | u16 num_msgs; |
813 | |
814 | af_mbox = container_of(work, struct mbox, mbox_wrk); |
815 | mbox = &af_mbox->mbox; |
816 | mdev = &mbox->dev[0]; |
817 | rsp_hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start); |
818 | num_msgs = rsp_hdr->num_msgs; |
819 | |
820 | offset = mbox->rx_start + ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN); |
821 | pf = af_mbox->pfvf; |
822 | |
823 | for (id = 0; id < num_msgs; id++) { |
824 | msg = (struct mbox_msghdr *)(mdev->mbase + offset); |
825 | otx2_process_pfaf_mbox_msg(pf, msg); |
826 | offset = mbox->rx_start + msg->next_msgoff; |
827 | if (mdev->msgs_acked == (num_msgs - 1)) |
828 | __otx2_mbox_reset(mbox, devid: 0); |
829 | mdev->msgs_acked++; |
830 | } |
831 | |
832 | } |
833 | |
834 | static void otx2_handle_link_event(struct otx2_nic *pf) |
835 | { |
836 | struct cgx_link_user_info *linfo = &pf->linfo; |
837 | struct net_device *netdev = pf->netdev; |
838 | |
839 | pr_info("%s NIC Link is %s %d Mbps %s duplex\n" , netdev->name, |
840 | linfo->link_up ? "UP" : "DOWN" , linfo->speed, |
841 | linfo->full_duplex ? "Full" : "Half" ); |
842 | if (linfo->link_up) { |
843 | netif_carrier_on(dev: netdev); |
844 | netif_tx_start_all_queues(dev: netdev); |
845 | } else { |
846 | netif_tx_stop_all_queues(dev: netdev); |
847 | netif_carrier_off(dev: netdev); |
848 | } |
849 | } |
850 | |
851 | int otx2_mbox_up_handler_mcs_intr_notify(struct otx2_nic *pf, |
852 | struct mcs_intr_info *event, |
853 | struct msg_rsp *rsp) |
854 | { |
855 | cn10k_handle_mcs_event(pfvf: pf, event); |
856 | |
857 | return 0; |
858 | } |
859 | |
860 | int otx2_mbox_up_handler_cgx_link_event(struct otx2_nic *pf, |
861 | struct cgx_link_info_msg *msg, |
862 | struct msg_rsp *rsp) |
863 | { |
864 | int i; |
865 | |
866 | /* Copy the link info sent by AF */ |
867 | pf->linfo = msg->link_info; |
868 | |
869 | /* notify VFs about link event */ |
870 | for (i = 0; i < pci_num_vf(dev: pf->pdev); i++) { |
871 | struct otx2_vf_config *config = &pf->vf_configs[i]; |
872 | struct delayed_work *dwork = &config->link_event_work; |
873 | |
874 | if (config->intf_down) |
875 | continue; |
876 | |
877 | schedule_delayed_work(dwork, delay: msecs_to_jiffies(m: 100)); |
878 | } |
879 | |
880 | /* interface has not been fully configured yet */ |
881 | if (pf->flags & OTX2_FLAG_INTF_DOWN) |
882 | return 0; |
883 | |
884 | otx2_handle_link_event(pf); |
885 | return 0; |
886 | } |
887 | |
888 | static int otx2_process_mbox_msg_up(struct otx2_nic *pf, |
889 | struct mbox_msghdr *req) |
890 | { |
891 | /* Check if valid, if not reply with a invalid msg */ |
892 | if (req->sig != OTX2_MBOX_REQ_SIG) { |
893 | otx2_reply_invalid_msg(mbox: &pf->mbox.mbox_up, devid: 0, pcifunc: 0, id: req->id); |
894 | return -ENODEV; |
895 | } |
896 | |
897 | switch (req->id) { |
898 | #define M(_name, _id, _fn_name, _req_type, _rsp_type) \ |
899 | case _id: { \ |
900 | struct _rsp_type *rsp; \ |
901 | int err; \ |
902 | \ |
903 | rsp = (struct _rsp_type *)otx2_mbox_alloc_msg( \ |
904 | &pf->mbox.mbox_up, 0, \ |
905 | sizeof(struct _rsp_type)); \ |
906 | if (!rsp) \ |
907 | return -ENOMEM; \ |
908 | \ |
909 | rsp->hdr.id = _id; \ |
910 | rsp->hdr.sig = OTX2_MBOX_RSP_SIG; \ |
911 | rsp->hdr.pcifunc = 0; \ |
912 | rsp->hdr.rc = 0; \ |
913 | \ |
914 | err = otx2_mbox_up_handler_ ## _fn_name( \ |
915 | pf, (struct _req_type *)req, rsp); \ |
916 | return err; \ |
917 | } |
918 | MBOX_UP_CGX_MESSAGES |
919 | MBOX_UP_MCS_MESSAGES |
920 | #undef M |
921 | break; |
922 | default: |
923 | otx2_reply_invalid_msg(mbox: &pf->mbox.mbox_up, devid: 0, pcifunc: 0, id: req->id); |
924 | return -ENODEV; |
925 | } |
926 | return 0; |
927 | } |
928 | |
929 | static void otx2_pfaf_mbox_up_handler(struct work_struct *work) |
930 | { |
931 | struct mbox *af_mbox = container_of(work, struct mbox, mbox_up_wrk); |
932 | struct otx2_mbox *mbox = &af_mbox->mbox_up; |
933 | struct otx2_mbox_dev *mdev = &mbox->dev[0]; |
934 | struct otx2_nic *pf = af_mbox->pfvf; |
935 | int offset, id, devid = 0; |
936 | struct mbox_hdr *rsp_hdr; |
937 | struct mbox_msghdr *msg; |
938 | u16 num_msgs; |
939 | |
940 | rsp_hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start); |
941 | num_msgs = rsp_hdr->num_msgs; |
942 | |
943 | offset = mbox->rx_start + ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN); |
944 | |
945 | for (id = 0; id < num_msgs; id++) { |
946 | msg = (struct mbox_msghdr *)(mdev->mbase + offset); |
947 | |
948 | devid = msg->pcifunc & RVU_PFVF_FUNC_MASK; |
949 | /* Skip processing VF's messages */ |
950 | if (!devid) |
951 | otx2_process_mbox_msg_up(pf, req: msg); |
952 | offset = mbox->rx_start + msg->next_msgoff; |
953 | } |
954 | /* Forward to VF iff VFs are really present */ |
955 | if (devid && pci_num_vf(dev: pf->pdev)) { |
956 | otx2_forward_vf_mbox_msgs(pf, src_mbox: &pf->mbox.mbox_up, |
957 | MBOX_DIR_PFVF_UP, vf: devid - 1, |
958 | num_msgs); |
959 | return; |
960 | } |
961 | |
962 | otx2_mbox_msg_send(mbox, devid: 0); |
963 | } |
964 | |
965 | static irqreturn_t otx2_pfaf_mbox_intr_handler(int irq, void *pf_irq) |
966 | { |
967 | struct otx2_nic *pf = (struct otx2_nic *)pf_irq; |
968 | struct mbox *mw = &pf->mbox; |
969 | struct otx2_mbox_dev *mdev; |
970 | struct otx2_mbox *mbox; |
971 | struct mbox_hdr *hdr; |
972 | u64 mbox_data; |
973 | |
974 | /* Clear the IRQ */ |
975 | otx2_write64(nic: pf, RVU_PF_INT, BIT_ULL(0)); |
976 | |
977 | |
978 | mbox_data = otx2_read64(nic: pf, RVU_PF_PFAF_MBOX0); |
979 | |
980 | if (mbox_data & MBOX_UP_MSG) { |
981 | mbox_data &= ~MBOX_UP_MSG; |
982 | otx2_write64(nic: pf, RVU_PF_PFAF_MBOX0, val: mbox_data); |
983 | |
984 | mbox = &mw->mbox_up; |
985 | mdev = &mbox->dev[0]; |
986 | otx2_sync_mbox_bbuf(mbox, devid: 0); |
987 | |
988 | hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start); |
989 | if (hdr->num_msgs) |
990 | queue_work(wq: pf->mbox_wq, work: &mw->mbox_up_wrk); |
991 | |
992 | trace_otx2_msg_interrupt(pdev: pf->pdev, msg: "UP message from AF to PF" , |
993 | BIT_ULL(0)); |
994 | } |
995 | |
996 | if (mbox_data & MBOX_DOWN_MSG) { |
997 | mbox_data &= ~MBOX_DOWN_MSG; |
998 | otx2_write64(nic: pf, RVU_PF_PFAF_MBOX0, val: mbox_data); |
999 | |
1000 | mbox = &mw->mbox; |
1001 | mdev = &mbox->dev[0]; |
1002 | otx2_sync_mbox_bbuf(mbox, devid: 0); |
1003 | |
1004 | hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start); |
1005 | if (hdr->num_msgs) |
1006 | queue_work(wq: pf->mbox_wq, work: &mw->mbox_wrk); |
1007 | |
1008 | trace_otx2_msg_interrupt(pdev: pf->pdev, msg: "DOWN reply from AF to PF" , |
1009 | BIT_ULL(0)); |
1010 | } |
1011 | |
1012 | return IRQ_HANDLED; |
1013 | } |
1014 | |
1015 | static void otx2_disable_mbox_intr(struct otx2_nic *pf) |
1016 | { |
1017 | int vector = pci_irq_vector(dev: pf->pdev, nr: RVU_PF_INT_VEC_AFPF_MBOX); |
1018 | |
1019 | /* Disable AF => PF mailbox IRQ */ |
1020 | otx2_write64(nic: pf, RVU_PF_INT_ENA_W1C, BIT_ULL(0)); |
1021 | free_irq(vector, pf); |
1022 | } |
1023 | |
1024 | static int otx2_register_mbox_intr(struct otx2_nic *pf, bool probe_af) |
1025 | { |
1026 | struct otx2_hw *hw = &pf->hw; |
1027 | struct msg_req *req; |
1028 | char *irq_name; |
1029 | int err; |
1030 | |
1031 | /* Register mailbox interrupt handler */ |
1032 | irq_name = &hw->irq_name[RVU_PF_INT_VEC_AFPF_MBOX * NAME_SIZE]; |
1033 | snprintf(buf: irq_name, NAME_SIZE, fmt: "RVUPFAF Mbox" ); |
1034 | err = request_irq(irq: pci_irq_vector(dev: pf->pdev, nr: RVU_PF_INT_VEC_AFPF_MBOX), |
1035 | handler: otx2_pfaf_mbox_intr_handler, flags: 0, name: irq_name, dev: pf); |
1036 | if (err) { |
1037 | dev_err(pf->dev, |
1038 | "RVUPF: IRQ registration failed for PFAF mbox irq\n" ); |
1039 | return err; |
1040 | } |
1041 | |
1042 | /* Enable mailbox interrupt for msgs coming from AF. |
1043 | * First clear to avoid spurious interrupts, if any. |
1044 | */ |
1045 | otx2_write64(nic: pf, RVU_PF_INT, BIT_ULL(0)); |
1046 | otx2_write64(nic: pf, RVU_PF_INT_ENA_W1S, BIT_ULL(0)); |
1047 | |
1048 | if (!probe_af) |
1049 | return 0; |
1050 | |
1051 | /* Check mailbox communication with AF */ |
1052 | req = otx2_mbox_alloc_msg_ready(mbox: &pf->mbox); |
1053 | if (!req) { |
1054 | otx2_disable_mbox_intr(pf); |
1055 | return -ENOMEM; |
1056 | } |
1057 | err = otx2_sync_mbox_msg(mbox: &pf->mbox); |
1058 | if (err) { |
1059 | dev_warn(pf->dev, |
1060 | "AF not responding to mailbox, deferring probe\n" ); |
1061 | otx2_disable_mbox_intr(pf); |
1062 | return -EPROBE_DEFER; |
1063 | } |
1064 | |
1065 | return 0; |
1066 | } |
1067 | |
1068 | static void otx2_pfaf_mbox_destroy(struct otx2_nic *pf) |
1069 | { |
1070 | struct mbox *mbox = &pf->mbox; |
1071 | |
1072 | if (pf->mbox_wq) { |
1073 | destroy_workqueue(wq: pf->mbox_wq); |
1074 | pf->mbox_wq = NULL; |
1075 | } |
1076 | |
1077 | if (mbox->mbox.hwbase) |
1078 | iounmap(addr: (void __iomem *)mbox->mbox.hwbase); |
1079 | |
1080 | otx2_mbox_destroy(mbox: &mbox->mbox); |
1081 | otx2_mbox_destroy(mbox: &mbox->mbox_up); |
1082 | } |
1083 | |
1084 | static int otx2_pfaf_mbox_init(struct otx2_nic *pf) |
1085 | { |
1086 | struct mbox *mbox = &pf->mbox; |
1087 | void __iomem *hwbase; |
1088 | int err; |
1089 | |
1090 | mbox->pfvf = pf; |
1091 | pf->mbox_wq = alloc_ordered_workqueue("otx2_pfaf_mailbox" , |
1092 | WQ_HIGHPRI | WQ_MEM_RECLAIM); |
1093 | if (!pf->mbox_wq) |
1094 | return -ENOMEM; |
1095 | |
1096 | /* Mailbox is a reserved memory (in RAM) region shared between |
1097 | * admin function (i.e AF) and this PF, shouldn't be mapped as |
1098 | * device memory to allow unaligned accesses. |
1099 | */ |
1100 | hwbase = ioremap_wc(pci_resource_start(pf->pdev, PCI_MBOX_BAR_NUM), |
1101 | MBOX_SIZE); |
1102 | if (!hwbase) { |
1103 | dev_err(pf->dev, "Unable to map PFAF mailbox region\n" ); |
1104 | err = -ENOMEM; |
1105 | goto exit; |
1106 | } |
1107 | |
1108 | err = otx2_mbox_init(mbox: &mbox->mbox, hwbase, pdev: pf->pdev, reg_base: pf->reg_base, |
1109 | MBOX_DIR_PFAF, ndevs: 1); |
1110 | if (err) |
1111 | goto exit; |
1112 | |
1113 | err = otx2_mbox_init(mbox: &mbox->mbox_up, hwbase, pdev: pf->pdev, reg_base: pf->reg_base, |
1114 | MBOX_DIR_PFAF_UP, ndevs: 1); |
1115 | if (err) |
1116 | goto exit; |
1117 | |
1118 | err = otx2_mbox_bbuf_init(mbox, pdev: pf->pdev); |
1119 | if (err) |
1120 | goto exit; |
1121 | |
1122 | INIT_WORK(&mbox->mbox_wrk, otx2_pfaf_mbox_handler); |
1123 | INIT_WORK(&mbox->mbox_up_wrk, otx2_pfaf_mbox_up_handler); |
1124 | mutex_init(&mbox->lock); |
1125 | |
1126 | return 0; |
1127 | exit: |
1128 | otx2_pfaf_mbox_destroy(pf); |
1129 | return err; |
1130 | } |
1131 | |
1132 | static int otx2_cgx_config_linkevents(struct otx2_nic *pf, bool enable) |
1133 | { |
1134 | struct msg_req *msg; |
1135 | int err; |
1136 | |
1137 | mutex_lock(&pf->mbox.lock); |
1138 | if (enable) |
1139 | msg = otx2_mbox_alloc_msg_cgx_start_linkevents(mbox: &pf->mbox); |
1140 | else |
1141 | msg = otx2_mbox_alloc_msg_cgx_stop_linkevents(mbox: &pf->mbox); |
1142 | |
1143 | if (!msg) { |
1144 | mutex_unlock(lock: &pf->mbox.lock); |
1145 | return -ENOMEM; |
1146 | } |
1147 | |
1148 | err = otx2_sync_mbox_msg(mbox: &pf->mbox); |
1149 | mutex_unlock(lock: &pf->mbox.lock); |
1150 | return err; |
1151 | } |
1152 | |
1153 | static int otx2_cgx_config_loopback(struct otx2_nic *pf, bool enable) |
1154 | { |
1155 | struct msg_req *msg; |
1156 | int err; |
1157 | |
1158 | if (enable && !bitmap_empty(src: pf->flow_cfg->dmacflt_bmap, |
1159 | nbits: pf->flow_cfg->dmacflt_max_flows)) |
1160 | netdev_warn(dev: pf->netdev, |
1161 | format: "CGX/RPM internal loopback might not work as DMAC filters are active\n" ); |
1162 | |
1163 | mutex_lock(&pf->mbox.lock); |
1164 | if (enable) |
1165 | msg = otx2_mbox_alloc_msg_cgx_intlbk_enable(mbox: &pf->mbox); |
1166 | else |
1167 | msg = otx2_mbox_alloc_msg_cgx_intlbk_disable(mbox: &pf->mbox); |
1168 | |
1169 | if (!msg) { |
1170 | mutex_unlock(lock: &pf->mbox.lock); |
1171 | return -ENOMEM; |
1172 | } |
1173 | |
1174 | err = otx2_sync_mbox_msg(mbox: &pf->mbox); |
1175 | mutex_unlock(lock: &pf->mbox.lock); |
1176 | return err; |
1177 | } |
1178 | |
1179 | int otx2_set_real_num_queues(struct net_device *netdev, |
1180 | int tx_queues, int rx_queues) |
1181 | { |
1182 | int err; |
1183 | |
1184 | err = netif_set_real_num_tx_queues(dev: netdev, txq: tx_queues); |
1185 | if (err) { |
1186 | netdev_err(dev: netdev, |
1187 | format: "Failed to set no of Tx queues: %d\n" , tx_queues); |
1188 | return err; |
1189 | } |
1190 | |
1191 | err = netif_set_real_num_rx_queues(dev: netdev, rxq: rx_queues); |
1192 | if (err) |
1193 | netdev_err(dev: netdev, |
1194 | format: "Failed to set no of Rx queues: %d\n" , rx_queues); |
1195 | return err; |
1196 | } |
1197 | EXPORT_SYMBOL(otx2_set_real_num_queues); |
1198 | |
1199 | static char *nix_sqoperr_e_str[NIX_SQOPERR_MAX] = { |
1200 | "NIX_SQOPERR_OOR" , |
1201 | "NIX_SQOPERR_CTX_FAULT" , |
1202 | "NIX_SQOPERR_CTX_POISON" , |
1203 | "NIX_SQOPERR_DISABLED" , |
1204 | "NIX_SQOPERR_SIZE_ERR" , |
1205 | "NIX_SQOPERR_OFLOW" , |
1206 | "NIX_SQOPERR_SQB_NULL" , |
1207 | "NIX_SQOPERR_SQB_FAULT" , |
1208 | "NIX_SQOPERR_SQE_SZ_ZERO" , |
1209 | }; |
1210 | |
1211 | static char *nix_mnqerr_e_str[NIX_MNQERR_MAX] = { |
1212 | "NIX_MNQERR_SQ_CTX_FAULT" , |
1213 | "NIX_MNQERR_SQ_CTX_POISON" , |
1214 | "NIX_MNQERR_SQB_FAULT" , |
1215 | "NIX_MNQERR_SQB_POISON" , |
1216 | "NIX_MNQERR_TOTAL_ERR" , |
1217 | "NIX_MNQERR_LSO_ERR" , |
1218 | "NIX_MNQERR_CQ_QUERY_ERR" , |
1219 | "NIX_MNQERR_MAX_SQE_SIZE_ERR" , |
1220 | "NIX_MNQERR_MAXLEN_ERR" , |
1221 | "NIX_MNQERR_SQE_SIZEM1_ZERO" , |
1222 | }; |
1223 | |
1224 | static char *nix_snd_status_e_str[NIX_SND_STATUS_MAX] = { |
1225 | [NIX_SND_STATUS_GOOD] = "NIX_SND_STATUS_GOOD" , |
1226 | [NIX_SND_STATUS_SQ_CTX_FAULT] = "NIX_SND_STATUS_SQ_CTX_FAULT" , |
1227 | [NIX_SND_STATUS_SQ_CTX_POISON] = "NIX_SND_STATUS_SQ_CTX_POISON" , |
1228 | [NIX_SND_STATUS_SQB_FAULT] = "NIX_SND_STATUS_SQB_FAULT" , |
1229 | [NIX_SND_STATUS_SQB_POISON] = "NIX_SND_STATUS_SQB_POISON" , |
1230 | [NIX_SND_STATUS_HDR_ERR] = "NIX_SND_STATUS_HDR_ERR" , |
1231 | [NIX_SND_STATUS_EXT_ERR] = "NIX_SND_STATUS_EXT_ERR" , |
1232 | [NIX_SND_STATUS_JUMP_FAULT] = "NIX_SND_STATUS_JUMP_FAULT" , |
1233 | [NIX_SND_STATUS_JUMP_POISON] = "NIX_SND_STATUS_JUMP_POISON" , |
1234 | [NIX_SND_STATUS_CRC_ERR] = "NIX_SND_STATUS_CRC_ERR" , |
1235 | [NIX_SND_STATUS_IMM_ERR] = "NIX_SND_STATUS_IMM_ERR" , |
1236 | [NIX_SND_STATUS_SG_ERR] = "NIX_SND_STATUS_SG_ERR" , |
1237 | [NIX_SND_STATUS_MEM_ERR] = "NIX_SND_STATUS_MEM_ERR" , |
1238 | [NIX_SND_STATUS_INVALID_SUBDC] = "NIX_SND_STATUS_INVALID_SUBDC" , |
1239 | [NIX_SND_STATUS_SUBDC_ORDER_ERR] = "NIX_SND_STATUS_SUBDC_ORDER_ERR" , |
1240 | [NIX_SND_STATUS_DATA_FAULT] = "NIX_SND_STATUS_DATA_FAULT" , |
1241 | [NIX_SND_STATUS_DATA_POISON] = "NIX_SND_STATUS_DATA_POISON" , |
1242 | [NIX_SND_STATUS_NPC_DROP_ACTION] = "NIX_SND_STATUS_NPC_DROP_ACTION" , |
1243 | [NIX_SND_STATUS_LOCK_VIOL] = "NIX_SND_STATUS_LOCK_VIOL" , |
1244 | [NIX_SND_STATUS_NPC_UCAST_CHAN_ERR] = "NIX_SND_STAT_NPC_UCAST_CHAN_ERR" , |
1245 | [NIX_SND_STATUS_NPC_MCAST_CHAN_ERR] = "NIX_SND_STAT_NPC_MCAST_CHAN_ERR" , |
1246 | [NIX_SND_STATUS_NPC_MCAST_ABORT] = "NIX_SND_STATUS_NPC_MCAST_ABORT" , |
1247 | [NIX_SND_STATUS_NPC_VTAG_PTR_ERR] = "NIX_SND_STATUS_NPC_VTAG_PTR_ERR" , |
1248 | [NIX_SND_STATUS_NPC_VTAG_SIZE_ERR] = "NIX_SND_STATUS_NPC_VTAG_SIZE_ERR" , |
1249 | [NIX_SND_STATUS_SEND_MEM_FAULT] = "NIX_SND_STATUS_SEND_MEM_FAULT" , |
1250 | [NIX_SND_STATUS_SEND_STATS_ERR] = "NIX_SND_STATUS_SEND_STATS_ERR" , |
1251 | }; |
1252 | |
1253 | static irqreturn_t otx2_q_intr_handler(int irq, void *data) |
1254 | { |
1255 | struct otx2_nic *pf = data; |
1256 | struct otx2_snd_queue *sq; |
1257 | u64 val, *ptr; |
1258 | u64 qidx = 0; |
1259 | |
1260 | /* CQ */ |
1261 | for (qidx = 0; qidx < pf->qset.cq_cnt; qidx++) { |
1262 | ptr = otx2_get_regaddr(nic: pf, NIX_LF_CQ_OP_INT); |
1263 | val = otx2_atomic64_add((qidx << 44), ptr); |
1264 | |
1265 | otx2_write64(nic: pf, NIX_LF_CQ_OP_INT, val: (qidx << 44) | |
1266 | (val & NIX_CQERRINT_BITS)); |
1267 | if (!(val & (NIX_CQERRINT_BITS | BIT_ULL(42)))) |
1268 | continue; |
1269 | |
1270 | if (val & BIT_ULL(42)) { |
1271 | netdev_err(dev: pf->netdev, |
1272 | format: "CQ%lld: error reading NIX_LF_CQ_OP_INT, NIX_LF_ERR_INT 0x%llx\n" , |
1273 | qidx, otx2_read64(nic: pf, NIX_LF_ERR_INT)); |
1274 | } else { |
1275 | if (val & BIT_ULL(NIX_CQERRINT_DOOR_ERR)) |
1276 | netdev_err(dev: pf->netdev, format: "CQ%lld: Doorbell error" , |
1277 | qidx); |
1278 | if (val & BIT_ULL(NIX_CQERRINT_CQE_FAULT)) |
1279 | netdev_err(dev: pf->netdev, |
1280 | format: "CQ%lld: Memory fault on CQE write to LLC/DRAM" , |
1281 | qidx); |
1282 | } |
1283 | |
1284 | schedule_work(work: &pf->reset_task); |
1285 | } |
1286 | |
1287 | /* SQ */ |
1288 | for (qidx = 0; qidx < otx2_get_total_tx_queues(pfvf: pf); qidx++) { |
1289 | u64 sq_op_err_dbg, mnq_err_dbg, snd_err_dbg; |
1290 | u8 sq_op_err_code, mnq_err_code, snd_err_code; |
1291 | |
1292 | sq = &pf->qset.sq[qidx]; |
1293 | if (!sq->sqb_ptrs) |
1294 | continue; |
1295 | |
1296 | /* Below debug registers captures first errors corresponding to |
1297 | * those registers. We don't have to check against SQ qid as |
1298 | * these are fatal errors. |
1299 | */ |
1300 | |
1301 | ptr = otx2_get_regaddr(nic: pf, NIX_LF_SQ_OP_INT); |
1302 | val = otx2_atomic64_add((qidx << 44), ptr); |
1303 | otx2_write64(nic: pf, NIX_LF_SQ_OP_INT, val: (qidx << 44) | |
1304 | (val & NIX_SQINT_BITS)); |
1305 | |
1306 | if (val & BIT_ULL(42)) { |
1307 | netdev_err(dev: pf->netdev, |
1308 | format: "SQ%lld: error reading NIX_LF_SQ_OP_INT, NIX_LF_ERR_INT 0x%llx\n" , |
1309 | qidx, otx2_read64(nic: pf, NIX_LF_ERR_INT)); |
1310 | goto done; |
1311 | } |
1312 | |
1313 | sq_op_err_dbg = otx2_read64(nic: pf, NIX_LF_SQ_OP_ERR_DBG); |
1314 | if (!(sq_op_err_dbg & BIT(44))) |
1315 | goto chk_mnq_err_dbg; |
1316 | |
1317 | sq_op_err_code = FIELD_GET(GENMASK(7, 0), sq_op_err_dbg); |
1318 | netdev_err(dev: pf->netdev, |
1319 | format: "SQ%lld: NIX_LF_SQ_OP_ERR_DBG(0x%llx) err=%s(%#x)\n" , |
1320 | qidx, sq_op_err_dbg, |
1321 | nix_sqoperr_e_str[sq_op_err_code], |
1322 | sq_op_err_code); |
1323 | |
1324 | otx2_write64(nic: pf, NIX_LF_SQ_OP_ERR_DBG, BIT_ULL(44)); |
1325 | |
1326 | if (sq_op_err_code == NIX_SQOPERR_SQB_NULL) |
1327 | goto chk_mnq_err_dbg; |
1328 | |
1329 | /* Err is not NIX_SQOPERR_SQB_NULL, call aq function to read SQ structure. |
1330 | * TODO: But we are in irq context. How to call mbox functions which does sleep |
1331 | */ |
1332 | |
1333 | chk_mnq_err_dbg: |
1334 | mnq_err_dbg = otx2_read64(nic: pf, NIX_LF_MNQ_ERR_DBG); |
1335 | if (!(mnq_err_dbg & BIT(44))) |
1336 | goto chk_snd_err_dbg; |
1337 | |
1338 | mnq_err_code = FIELD_GET(GENMASK(7, 0), mnq_err_dbg); |
1339 | netdev_err(dev: pf->netdev, |
1340 | format: "SQ%lld: NIX_LF_MNQ_ERR_DBG(0x%llx) err=%s(%#x)\n" , |
1341 | qidx, mnq_err_dbg, nix_mnqerr_e_str[mnq_err_code], |
1342 | mnq_err_code); |
1343 | otx2_write64(nic: pf, NIX_LF_MNQ_ERR_DBG, BIT_ULL(44)); |
1344 | |
1345 | chk_snd_err_dbg: |
1346 | snd_err_dbg = otx2_read64(nic: pf, NIX_LF_SEND_ERR_DBG); |
1347 | if (snd_err_dbg & BIT(44)) { |
1348 | snd_err_code = FIELD_GET(GENMASK(7, 0), snd_err_dbg); |
1349 | netdev_err(dev: pf->netdev, |
1350 | format: "SQ%lld: NIX_LF_SND_ERR_DBG:0x%llx err=%s(%#x)\n" , |
1351 | qidx, snd_err_dbg, |
1352 | nix_snd_status_e_str[snd_err_code], |
1353 | snd_err_code); |
1354 | otx2_write64(nic: pf, NIX_LF_SEND_ERR_DBG, BIT_ULL(44)); |
1355 | } |
1356 | |
1357 | done: |
1358 | /* Print values and reset */ |
1359 | if (val & BIT_ULL(NIX_SQINT_SQB_ALLOC_FAIL)) |
1360 | netdev_err(dev: pf->netdev, format: "SQ%lld: SQB allocation failed" , |
1361 | qidx); |
1362 | |
1363 | schedule_work(work: &pf->reset_task); |
1364 | } |
1365 | |
1366 | return IRQ_HANDLED; |
1367 | } |
1368 | |
1369 | static irqreturn_t otx2_cq_intr_handler(int irq, void *cq_irq) |
1370 | { |
1371 | struct otx2_cq_poll *cq_poll = (struct otx2_cq_poll *)cq_irq; |
1372 | struct otx2_nic *pf = (struct otx2_nic *)cq_poll->dev; |
1373 | int qidx = cq_poll->cint_idx; |
1374 | |
1375 | /* Disable interrupts. |
1376 | * |
1377 | * Completion interrupts behave in a level-triggered interrupt |
1378 | * fashion, and hence have to be cleared only after it is serviced. |
1379 | */ |
1380 | otx2_write64(nic: pf, NIX_LF_CINTX_ENA_W1C(qidx), BIT_ULL(0)); |
1381 | |
1382 | /* Schedule NAPI */ |
1383 | pf->napi_events++; |
1384 | napi_schedule_irqoff(n: &cq_poll->napi); |
1385 | |
1386 | return IRQ_HANDLED; |
1387 | } |
1388 | |
1389 | static void otx2_disable_napi(struct otx2_nic *pf) |
1390 | { |
1391 | struct otx2_qset *qset = &pf->qset; |
1392 | struct otx2_cq_poll *cq_poll; |
1393 | int qidx; |
1394 | |
1395 | for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) { |
1396 | cq_poll = &qset->napi[qidx]; |
1397 | cancel_work_sync(work: &cq_poll->dim.work); |
1398 | napi_disable(n: &cq_poll->napi); |
1399 | netif_napi_del(napi: &cq_poll->napi); |
1400 | } |
1401 | } |
1402 | |
1403 | static void otx2_free_cq_res(struct otx2_nic *pf) |
1404 | { |
1405 | struct otx2_qset *qset = &pf->qset; |
1406 | struct otx2_cq_queue *cq; |
1407 | int qidx; |
1408 | |
1409 | /* Disable CQs */ |
1410 | otx2_ctx_disable(mbox: &pf->mbox, type: NIX_AQ_CTYPE_CQ, npa: false); |
1411 | for (qidx = 0; qidx < qset->cq_cnt; qidx++) { |
1412 | cq = &qset->cq[qidx]; |
1413 | qmem_free(dev: pf->dev, qmem: cq->cqe); |
1414 | } |
1415 | } |
1416 | |
1417 | static void otx2_free_sq_res(struct otx2_nic *pf) |
1418 | { |
1419 | struct otx2_qset *qset = &pf->qset; |
1420 | struct otx2_snd_queue *sq; |
1421 | int qidx; |
1422 | |
1423 | /* Disable SQs */ |
1424 | otx2_ctx_disable(mbox: &pf->mbox, type: NIX_AQ_CTYPE_SQ, npa: false); |
1425 | /* Free SQB pointers */ |
1426 | otx2_sq_free_sqbs(pfvf: pf); |
1427 | for (qidx = 0; qidx < otx2_get_total_tx_queues(pfvf: pf); qidx++) { |
1428 | sq = &qset->sq[qidx]; |
1429 | /* Skip freeing Qos queues if they are not initialized */ |
1430 | if (!sq->sqe) |
1431 | continue; |
1432 | qmem_free(dev: pf->dev, qmem: sq->sqe); |
1433 | qmem_free(dev: pf->dev, qmem: sq->tso_hdrs); |
1434 | kfree(objp: sq->sg); |
1435 | kfree(objp: sq->sqb_ptrs); |
1436 | } |
1437 | } |
1438 | |
1439 | static int otx2_get_rbuf_size(struct otx2_nic *pf, int mtu) |
1440 | { |
1441 | int frame_size; |
1442 | int total_size; |
1443 | int rbuf_size; |
1444 | |
1445 | if (pf->hw.rbuf_len) |
1446 | return ALIGN(pf->hw.rbuf_len, OTX2_ALIGN) + OTX2_HEAD_ROOM; |
1447 | |
1448 | /* The data transferred by NIX to memory consists of actual packet |
1449 | * plus additional data which has timestamp and/or EDSA/HIGIG2 |
1450 | * headers if interface is configured in corresponding modes. |
1451 | * NIX transfers entire data using 6 segments/buffers and writes |
1452 | * a CQE_RX descriptor with those segment addresses. First segment |
1453 | * has additional data prepended to packet. Also software omits a |
1454 | * headroom of 128 bytes in each segment. Hence the total size of |
1455 | * memory needed to receive a packet with 'mtu' is: |
1456 | * frame size = mtu + additional data; |
1457 | * memory = frame_size + headroom * 6; |
1458 | * each receive buffer size = memory / 6; |
1459 | */ |
1460 | frame_size = mtu + OTX2_ETH_HLEN + OTX2_HW_TIMESTAMP_LEN; |
1461 | total_size = frame_size + OTX2_HEAD_ROOM * 6; |
1462 | rbuf_size = total_size / 6; |
1463 | |
1464 | return ALIGN(rbuf_size, 2048); |
1465 | } |
1466 | |
1467 | static int otx2_init_hw_resources(struct otx2_nic *pf) |
1468 | { |
1469 | struct nix_lf_free_req *free_req; |
1470 | struct mbox *mbox = &pf->mbox; |
1471 | struct otx2_hw *hw = &pf->hw; |
1472 | struct msg_req *req; |
1473 | int err = 0, lvl; |
1474 | |
1475 | /* Set required NPA LF's pool counts |
1476 | * Auras and Pools are used in a 1:1 mapping, |
1477 | * so, aura count = pool count. |
1478 | */ |
1479 | hw->rqpool_cnt = hw->rx_queues; |
1480 | hw->sqpool_cnt = otx2_get_total_tx_queues(pfvf: pf); |
1481 | hw->pool_cnt = hw->rqpool_cnt + hw->sqpool_cnt; |
1482 | |
1483 | /* Maximum hardware supported transmit length */ |
1484 | pf->tx_max_pktlen = pf->netdev->max_mtu + OTX2_ETH_HLEN; |
1485 | |
1486 | pf->rbsize = otx2_get_rbuf_size(pf, mtu: pf->netdev->mtu); |
1487 | |
1488 | mutex_lock(&mbox->lock); |
1489 | /* NPA init */ |
1490 | err = otx2_config_npa(pfvf: pf); |
1491 | if (err) |
1492 | goto exit; |
1493 | |
1494 | /* NIX init */ |
1495 | err = otx2_config_nix(pfvf: pf); |
1496 | if (err) |
1497 | goto err_free_npa_lf; |
1498 | |
1499 | /* Enable backpressure for CGX mapped PF/VFs */ |
1500 | if (!is_otx2_lbkvf(pdev: pf->pdev)) |
1501 | otx2_nix_config_bp(pfvf: pf, enable: true); |
1502 | |
1503 | /* Init Auras and pools used by NIX RQ, for free buffer ptrs */ |
1504 | err = otx2_rq_aura_pool_init(pfvf: pf); |
1505 | if (err) { |
1506 | mutex_unlock(lock: &mbox->lock); |
1507 | goto err_free_nix_lf; |
1508 | } |
1509 | /* Init Auras and pools used by NIX SQ, for queueing SQEs */ |
1510 | err = otx2_sq_aura_pool_init(pfvf: pf); |
1511 | if (err) { |
1512 | mutex_unlock(lock: &mbox->lock); |
1513 | goto err_free_rq_ptrs; |
1514 | } |
1515 | |
1516 | err = otx2_txsch_alloc(pfvf: pf); |
1517 | if (err) { |
1518 | mutex_unlock(lock: &mbox->lock); |
1519 | goto err_free_sq_ptrs; |
1520 | } |
1521 | |
1522 | #ifdef CONFIG_DCB |
1523 | if (pf->pfc_en) { |
1524 | err = otx2_pfc_txschq_alloc(pfvf: pf); |
1525 | if (err) { |
1526 | mutex_unlock(lock: &mbox->lock); |
1527 | goto err_free_sq_ptrs; |
1528 | } |
1529 | } |
1530 | #endif |
1531 | |
1532 | err = otx2_config_nix_queues(pfvf: pf); |
1533 | if (err) { |
1534 | mutex_unlock(lock: &mbox->lock); |
1535 | goto err_free_txsch; |
1536 | } |
1537 | |
1538 | for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { |
1539 | err = otx2_txschq_config(pfvf: pf, lvl, prio: 0, pfc_en: false); |
1540 | if (err) { |
1541 | mutex_unlock(lock: &mbox->lock); |
1542 | goto err_free_nix_queues; |
1543 | } |
1544 | } |
1545 | |
1546 | #ifdef CONFIG_DCB |
1547 | if (pf->pfc_en) { |
1548 | err = otx2_pfc_txschq_config(pfvf: pf); |
1549 | if (err) { |
1550 | mutex_unlock(lock: &mbox->lock); |
1551 | goto err_free_nix_queues; |
1552 | } |
1553 | } |
1554 | #endif |
1555 | |
1556 | mutex_unlock(lock: &mbox->lock); |
1557 | return err; |
1558 | |
1559 | err_free_nix_queues: |
1560 | otx2_free_sq_res(pf); |
1561 | otx2_free_cq_res(pf); |
1562 | otx2_ctx_disable(mbox, type: NIX_AQ_CTYPE_RQ, npa: false); |
1563 | err_free_txsch: |
1564 | otx2_txschq_stop(pfvf: pf); |
1565 | err_free_sq_ptrs: |
1566 | otx2_sq_free_sqbs(pfvf: pf); |
1567 | err_free_rq_ptrs: |
1568 | otx2_free_aura_ptr(pfvf: pf, type: AURA_NIX_RQ); |
1569 | otx2_ctx_disable(mbox, type: NPA_AQ_CTYPE_POOL, npa: true); |
1570 | otx2_ctx_disable(mbox, type: NPA_AQ_CTYPE_AURA, npa: true); |
1571 | otx2_aura_pool_free(pfvf: pf); |
1572 | err_free_nix_lf: |
1573 | mutex_lock(&mbox->lock); |
1574 | free_req = otx2_mbox_alloc_msg_nix_lf_free(mbox); |
1575 | if (free_req) { |
1576 | free_req->flags = NIX_LF_DISABLE_FLOWS; |
1577 | if (otx2_sync_mbox_msg(mbox)) |
1578 | dev_err(pf->dev, "%s failed to free nixlf\n" , __func__); |
1579 | } |
1580 | err_free_npa_lf: |
1581 | /* Reset NPA LF */ |
1582 | req = otx2_mbox_alloc_msg_npa_lf_free(mbox); |
1583 | if (req) { |
1584 | if (otx2_sync_mbox_msg(mbox)) |
1585 | dev_err(pf->dev, "%s failed to free npalf\n" , __func__); |
1586 | } |
1587 | exit: |
1588 | mutex_unlock(lock: &mbox->lock); |
1589 | return err; |
1590 | } |
1591 | |
1592 | static void otx2_free_hw_resources(struct otx2_nic *pf) |
1593 | { |
1594 | struct otx2_qset *qset = &pf->qset; |
1595 | struct nix_lf_free_req *free_req; |
1596 | struct mbox *mbox = &pf->mbox; |
1597 | struct otx2_cq_queue *cq; |
1598 | struct otx2_pool *pool; |
1599 | struct msg_req *req; |
1600 | int pool_id; |
1601 | int qidx; |
1602 | |
1603 | /* Ensure all SQE are processed */ |
1604 | otx2_sqb_flush(pfvf: pf); |
1605 | |
1606 | /* Stop transmission */ |
1607 | otx2_txschq_stop(pfvf: pf); |
1608 | |
1609 | #ifdef CONFIG_DCB |
1610 | if (pf->pfc_en) |
1611 | otx2_pfc_txschq_stop(pfvf: pf); |
1612 | #endif |
1613 | |
1614 | otx2_clean_qos_queues(pfvf: pf); |
1615 | |
1616 | mutex_lock(&mbox->lock); |
1617 | /* Disable backpressure */ |
1618 | if (!(pf->pcifunc & RVU_PFVF_FUNC_MASK)) |
1619 | otx2_nix_config_bp(pfvf: pf, enable: false); |
1620 | mutex_unlock(lock: &mbox->lock); |
1621 | |
1622 | /* Disable RQs */ |
1623 | otx2_ctx_disable(mbox, type: NIX_AQ_CTYPE_RQ, npa: false); |
1624 | |
1625 | /*Dequeue all CQEs */ |
1626 | for (qidx = 0; qidx < qset->cq_cnt; qidx++) { |
1627 | cq = &qset->cq[qidx]; |
1628 | if (cq->cq_type == CQ_RX) |
1629 | otx2_cleanup_rx_cqes(pfvf: pf, cq, qidx); |
1630 | else |
1631 | otx2_cleanup_tx_cqes(pfvf: pf, cq); |
1632 | } |
1633 | otx2_free_pending_sqe(pfvf: pf); |
1634 | |
1635 | otx2_free_sq_res(pf); |
1636 | |
1637 | /* Free RQ buffer pointers*/ |
1638 | otx2_free_aura_ptr(pfvf: pf, type: AURA_NIX_RQ); |
1639 | |
1640 | for (qidx = 0; qidx < pf->hw.rx_queues; qidx++) { |
1641 | pool_id = otx2_get_pool_idx(pfvf: pf, type: AURA_NIX_RQ, idx: qidx); |
1642 | pool = &pf->qset.pool[pool_id]; |
1643 | page_pool_destroy(pool: pool->page_pool); |
1644 | pool->page_pool = NULL; |
1645 | } |
1646 | |
1647 | otx2_free_cq_res(pf); |
1648 | |
1649 | /* Free all ingress bandwidth profiles allocated */ |
1650 | cn10k_free_all_ipolicers(pfvf: pf); |
1651 | |
1652 | mutex_lock(&mbox->lock); |
1653 | /* Reset NIX LF */ |
1654 | free_req = otx2_mbox_alloc_msg_nix_lf_free(mbox); |
1655 | if (free_req) { |
1656 | free_req->flags = NIX_LF_DISABLE_FLOWS; |
1657 | if (!(pf->flags & OTX2_FLAG_PF_SHUTDOWN)) |
1658 | free_req->flags |= NIX_LF_DONT_FREE_TX_VTAG; |
1659 | if (otx2_sync_mbox_msg(mbox)) |
1660 | dev_err(pf->dev, "%s failed to free nixlf\n" , __func__); |
1661 | } |
1662 | mutex_unlock(lock: &mbox->lock); |
1663 | |
1664 | /* Disable NPA Pool and Aura hw context */ |
1665 | otx2_ctx_disable(mbox, type: NPA_AQ_CTYPE_POOL, npa: true); |
1666 | otx2_ctx_disable(mbox, type: NPA_AQ_CTYPE_AURA, npa: true); |
1667 | otx2_aura_pool_free(pfvf: pf); |
1668 | |
1669 | mutex_lock(&mbox->lock); |
1670 | /* Reset NPA LF */ |
1671 | req = otx2_mbox_alloc_msg_npa_lf_free(mbox); |
1672 | if (req) { |
1673 | if (otx2_sync_mbox_msg(mbox)) |
1674 | dev_err(pf->dev, "%s failed to free npalf\n" , __func__); |
1675 | } |
1676 | mutex_unlock(lock: &mbox->lock); |
1677 | } |
1678 | |
1679 | static bool otx2_promisc_use_mce_list(struct otx2_nic *pfvf) |
1680 | { |
1681 | int vf; |
1682 | |
1683 | /* The AF driver will determine whether to allow the VF netdev or not */ |
1684 | if (is_otx2_vf(pcifunc: pfvf->pcifunc)) |
1685 | return true; |
1686 | |
1687 | /* check if there are any trusted VFs associated with the PF netdev */ |
1688 | for (vf = 0; vf < pci_num_vf(dev: pfvf->pdev); vf++) |
1689 | if (pfvf->vf_configs[vf].trusted) |
1690 | return true; |
1691 | return false; |
1692 | } |
1693 | |
1694 | static void otx2_do_set_rx_mode(struct otx2_nic *pf) |
1695 | { |
1696 | struct net_device *netdev = pf->netdev; |
1697 | struct nix_rx_mode *req; |
1698 | bool promisc = false; |
1699 | |
1700 | if (!(netdev->flags & IFF_UP)) |
1701 | return; |
1702 | |
1703 | if ((netdev->flags & IFF_PROMISC) || |
1704 | (netdev_uc_count(netdev) > OTX2_MAX_UNICAST_FLOWS)) { |
1705 | promisc = true; |
1706 | } |
1707 | |
1708 | /* Write unicast address to mcam entries or del from mcam */ |
1709 | if (!promisc && netdev->priv_flags & IFF_UNICAST_FLT) |
1710 | __dev_uc_sync(dev: netdev, sync: otx2_add_macfilter, unsync: otx2_del_macfilter); |
1711 | |
1712 | mutex_lock(&pf->mbox.lock); |
1713 | req = otx2_mbox_alloc_msg_nix_set_rx_mode(mbox: &pf->mbox); |
1714 | if (!req) { |
1715 | mutex_unlock(lock: &pf->mbox.lock); |
1716 | return; |
1717 | } |
1718 | |
1719 | req->mode = NIX_RX_MODE_UCAST; |
1720 | |
1721 | if (promisc) |
1722 | req->mode |= NIX_RX_MODE_PROMISC; |
1723 | if (netdev->flags & (IFF_ALLMULTI | IFF_MULTICAST)) |
1724 | req->mode |= NIX_RX_MODE_ALLMULTI; |
1725 | |
1726 | if (otx2_promisc_use_mce_list(pfvf: pf)) |
1727 | req->mode |= NIX_RX_MODE_USE_MCE; |
1728 | |
1729 | otx2_sync_mbox_msg(mbox: &pf->mbox); |
1730 | mutex_unlock(lock: &pf->mbox.lock); |
1731 | } |
1732 | |
1733 | static void otx2_set_irq_coalesce(struct otx2_nic *pfvf) |
1734 | { |
1735 | int cint; |
1736 | |
1737 | for (cint = 0; cint < pfvf->hw.cint_cnt; cint++) |
1738 | otx2_config_irq_coalescing(pfvf, qidx: cint); |
1739 | } |
1740 | |
1741 | static void otx2_dim_work(struct work_struct *w) |
1742 | { |
1743 | struct dim_cq_moder cur_moder; |
1744 | struct otx2_cq_poll *cq_poll; |
1745 | struct otx2_nic *pfvf; |
1746 | struct dim *dim; |
1747 | |
1748 | dim = container_of(w, struct dim, work); |
1749 | cur_moder = net_dim_get_rx_moderation(cq_period_mode: dim->mode, ix: dim->profile_ix); |
1750 | cq_poll = container_of(dim, struct otx2_cq_poll, dim); |
1751 | pfvf = (struct otx2_nic *)cq_poll->dev; |
1752 | pfvf->hw.cq_time_wait = (cur_moder.usec > CQ_TIMER_THRESH_MAX) ? |
1753 | CQ_TIMER_THRESH_MAX : cur_moder.usec; |
1754 | pfvf->hw.cq_ecount_wait = (cur_moder.pkts > NAPI_POLL_WEIGHT) ? |
1755 | NAPI_POLL_WEIGHT : cur_moder.pkts; |
1756 | otx2_set_irq_coalesce(pfvf); |
1757 | dim->state = DIM_START_MEASURE; |
1758 | } |
1759 | |
1760 | int otx2_open(struct net_device *netdev) |
1761 | { |
1762 | struct otx2_nic *pf = netdev_priv(dev: netdev); |
1763 | struct otx2_cq_poll *cq_poll = NULL; |
1764 | struct otx2_qset *qset = &pf->qset; |
1765 | int err = 0, qidx, vec; |
1766 | char *irq_name; |
1767 | |
1768 | netif_carrier_off(dev: netdev); |
1769 | |
1770 | /* RQ and SQs are mapped to different CQs, |
1771 | * so find out max CQ IRQs (i.e CINTs) needed. |
1772 | */ |
1773 | pf->hw.non_qos_queues = pf->hw.tx_queues + pf->hw.xdp_queues; |
1774 | pf->hw.cint_cnt = max3(pf->hw.rx_queues, pf->hw.tx_queues, |
1775 | pf->hw.tc_tx_queues); |
1776 | |
1777 | pf->qset.cq_cnt = pf->hw.rx_queues + otx2_get_total_tx_queues(pfvf: pf); |
1778 | |
1779 | qset->napi = kcalloc(n: pf->hw.cint_cnt, size: sizeof(*cq_poll), GFP_KERNEL); |
1780 | if (!qset->napi) |
1781 | return -ENOMEM; |
1782 | |
1783 | /* CQ size of RQ */ |
1784 | qset->rqe_cnt = qset->rqe_cnt ? qset->rqe_cnt : Q_COUNT(Q_SIZE_256); |
1785 | /* CQ size of SQ */ |
1786 | qset->sqe_cnt = qset->sqe_cnt ? qset->sqe_cnt : Q_COUNT(Q_SIZE_4K); |
1787 | |
1788 | err = -ENOMEM; |
1789 | qset->cq = kcalloc(n: pf->qset.cq_cnt, |
1790 | size: sizeof(struct otx2_cq_queue), GFP_KERNEL); |
1791 | if (!qset->cq) |
1792 | goto err_free_mem; |
1793 | |
1794 | qset->sq = kcalloc(n: otx2_get_total_tx_queues(pfvf: pf), |
1795 | size: sizeof(struct otx2_snd_queue), GFP_KERNEL); |
1796 | if (!qset->sq) |
1797 | goto err_free_mem; |
1798 | |
1799 | qset->rq = kcalloc(n: pf->hw.rx_queues, |
1800 | size: sizeof(struct otx2_rcv_queue), GFP_KERNEL); |
1801 | if (!qset->rq) |
1802 | goto err_free_mem; |
1803 | |
1804 | err = otx2_init_hw_resources(pf); |
1805 | if (err) |
1806 | goto err_free_mem; |
1807 | |
1808 | /* Register NAPI handler */ |
1809 | for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) { |
1810 | cq_poll = &qset->napi[qidx]; |
1811 | cq_poll->cint_idx = qidx; |
1812 | /* RQ0 & SQ0 are mapped to CINT0 and so on.. |
1813 | * 'cq_ids[0]' points to RQ's CQ and |
1814 | * 'cq_ids[1]' points to SQ's CQ and |
1815 | * 'cq_ids[2]' points to XDP's CQ and |
1816 | */ |
1817 | cq_poll->cq_ids[CQ_RX] = |
1818 | (qidx < pf->hw.rx_queues) ? qidx : CINT_INVALID_CQ; |
1819 | cq_poll->cq_ids[CQ_TX] = (qidx < pf->hw.tx_queues) ? |
1820 | qidx + pf->hw.rx_queues : CINT_INVALID_CQ; |
1821 | if (pf->xdp_prog) |
1822 | cq_poll->cq_ids[CQ_XDP] = (qidx < pf->hw.xdp_queues) ? |
1823 | (qidx + pf->hw.rx_queues + |
1824 | pf->hw.tx_queues) : |
1825 | CINT_INVALID_CQ; |
1826 | else |
1827 | cq_poll->cq_ids[CQ_XDP] = CINT_INVALID_CQ; |
1828 | |
1829 | cq_poll->cq_ids[CQ_QOS] = (qidx < pf->hw.tc_tx_queues) ? |
1830 | (qidx + pf->hw.rx_queues + |
1831 | pf->hw.non_qos_queues) : |
1832 | CINT_INVALID_CQ; |
1833 | |
1834 | cq_poll->dev = (void *)pf; |
1835 | cq_poll->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE; |
1836 | INIT_WORK(&cq_poll->dim.work, otx2_dim_work); |
1837 | netif_napi_add(dev: netdev, napi: &cq_poll->napi, poll: otx2_napi_handler); |
1838 | napi_enable(n: &cq_poll->napi); |
1839 | } |
1840 | |
1841 | /* Set maximum frame size allowed in HW */ |
1842 | err = otx2_hw_set_mtu(pfvf: pf, mtu: netdev->mtu); |
1843 | if (err) |
1844 | goto err_disable_napi; |
1845 | |
1846 | /* Setup segmentation algorithms, if failed, clear offload capability */ |
1847 | otx2_setup_segmentation(pfvf: pf); |
1848 | |
1849 | /* Initialize RSS */ |
1850 | err = otx2_rss_init(pfvf: pf); |
1851 | if (err) |
1852 | goto err_disable_napi; |
1853 | |
1854 | /* Register Queue IRQ handlers */ |
1855 | vec = pf->hw.nix_msixoff + NIX_LF_QINT_VEC_START; |
1856 | irq_name = &pf->hw.irq_name[vec * NAME_SIZE]; |
1857 | |
1858 | snprintf(buf: irq_name, NAME_SIZE, fmt: "%s-qerr" , pf->netdev->name); |
1859 | |
1860 | err = request_irq(irq: pci_irq_vector(dev: pf->pdev, nr: vec), |
1861 | handler: otx2_q_intr_handler, flags: 0, name: irq_name, dev: pf); |
1862 | if (err) { |
1863 | dev_err(pf->dev, |
1864 | "RVUPF%d: IRQ registration failed for QERR\n" , |
1865 | rvu_get_pf(pf->pcifunc)); |
1866 | goto err_disable_napi; |
1867 | } |
1868 | |
1869 | /* Enable QINT IRQ */ |
1870 | otx2_write64(nic: pf, NIX_LF_QINTX_ENA_W1S(0), BIT_ULL(0)); |
1871 | |
1872 | /* Register CQ IRQ handlers */ |
1873 | vec = pf->hw.nix_msixoff + NIX_LF_CINT_VEC_START; |
1874 | for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) { |
1875 | irq_name = &pf->hw.irq_name[vec * NAME_SIZE]; |
1876 | |
1877 | snprintf(buf: irq_name, NAME_SIZE, fmt: "%s-rxtx-%d" , pf->netdev->name, |
1878 | qidx); |
1879 | |
1880 | err = request_irq(irq: pci_irq_vector(dev: pf->pdev, nr: vec), |
1881 | handler: otx2_cq_intr_handler, flags: 0, name: irq_name, |
1882 | dev: &qset->napi[qidx]); |
1883 | if (err) { |
1884 | dev_err(pf->dev, |
1885 | "RVUPF%d: IRQ registration failed for CQ%d\n" , |
1886 | rvu_get_pf(pf->pcifunc), qidx); |
1887 | goto err_free_cints; |
1888 | } |
1889 | vec++; |
1890 | |
1891 | otx2_config_irq_coalescing(pfvf: pf, qidx); |
1892 | |
1893 | /* Enable CQ IRQ */ |
1894 | otx2_write64(nic: pf, NIX_LF_CINTX_INT(qidx), BIT_ULL(0)); |
1895 | otx2_write64(nic: pf, NIX_LF_CINTX_ENA_W1S(qidx), BIT_ULL(0)); |
1896 | } |
1897 | |
1898 | otx2_set_cints_affinity(pfvf: pf); |
1899 | |
1900 | if (pf->flags & OTX2_FLAG_RX_VLAN_SUPPORT) |
1901 | otx2_enable_rxvlan(pf, enable: true); |
1902 | |
1903 | /* When reinitializing enable time stamping if it is enabled before */ |
1904 | if (pf->flags & OTX2_FLAG_TX_TSTAMP_ENABLED) { |
1905 | pf->flags &= ~OTX2_FLAG_TX_TSTAMP_ENABLED; |
1906 | otx2_config_hw_tx_tstamp(pfvf: pf, enable: true); |
1907 | } |
1908 | if (pf->flags & OTX2_FLAG_RX_TSTAMP_ENABLED) { |
1909 | pf->flags &= ~OTX2_FLAG_RX_TSTAMP_ENABLED; |
1910 | otx2_config_hw_rx_tstamp(pfvf: pf, enable: true); |
1911 | } |
1912 | |
1913 | pf->flags &= ~OTX2_FLAG_INTF_DOWN; |
1914 | /* 'intf_down' may be checked on any cpu */ |
1915 | smp_wmb(); |
1916 | |
1917 | /* Enable QoS configuration before starting tx queues */ |
1918 | otx2_qos_config_txschq(pfvf: pf); |
1919 | |
1920 | /* we have already received link status notification */ |
1921 | if (pf->linfo.link_up && !(pf->pcifunc & RVU_PFVF_FUNC_MASK)) |
1922 | otx2_handle_link_event(pf); |
1923 | |
1924 | /* Install DMAC Filters */ |
1925 | if (pf->flags & OTX2_FLAG_DMACFLTR_SUPPORT) |
1926 | otx2_dmacflt_reinstall_flows(pf); |
1927 | |
1928 | otx2_tc_apply_ingress_police_rules(nic: pf); |
1929 | |
1930 | err = otx2_rxtx_enable(pfvf: pf, enable: true); |
1931 | /* If a mbox communication error happens at this point then interface |
1932 | * will end up in a state such that it is in down state but hardware |
1933 | * mcam entries are enabled to receive the packets. Hence disable the |
1934 | * packet I/O. |
1935 | */ |
1936 | if (err == -EIO) |
1937 | goto err_disable_rxtx; |
1938 | else if (err) |
1939 | goto err_tx_stop_queues; |
1940 | |
1941 | otx2_do_set_rx_mode(pf); |
1942 | |
1943 | return 0; |
1944 | |
1945 | err_disable_rxtx: |
1946 | otx2_rxtx_enable(pfvf: pf, enable: false); |
1947 | err_tx_stop_queues: |
1948 | netif_tx_stop_all_queues(dev: netdev); |
1949 | netif_carrier_off(dev: netdev); |
1950 | pf->flags |= OTX2_FLAG_INTF_DOWN; |
1951 | err_free_cints: |
1952 | otx2_free_cints(pfvf: pf, n: qidx); |
1953 | vec = pci_irq_vector(dev: pf->pdev, |
1954 | nr: pf->hw.nix_msixoff + NIX_LF_QINT_VEC_START); |
1955 | otx2_write64(nic: pf, NIX_LF_QINTX_ENA_W1C(0), BIT_ULL(0)); |
1956 | free_irq(vec, pf); |
1957 | err_disable_napi: |
1958 | otx2_disable_napi(pf); |
1959 | otx2_free_hw_resources(pf); |
1960 | err_free_mem: |
1961 | kfree(objp: qset->sq); |
1962 | kfree(objp: qset->cq); |
1963 | kfree(objp: qset->rq); |
1964 | kfree(objp: qset->napi); |
1965 | return err; |
1966 | } |
1967 | EXPORT_SYMBOL(otx2_open); |
1968 | |
1969 | int otx2_stop(struct net_device *netdev) |
1970 | { |
1971 | struct otx2_nic *pf = netdev_priv(dev: netdev); |
1972 | struct otx2_cq_poll *cq_poll = NULL; |
1973 | struct otx2_qset *qset = &pf->qset; |
1974 | struct otx2_rss_info *; |
1975 | int qidx, vec, wrk; |
1976 | |
1977 | /* If the DOWN flag is set resources are already freed */ |
1978 | if (pf->flags & OTX2_FLAG_INTF_DOWN) |
1979 | return 0; |
1980 | |
1981 | netif_carrier_off(dev: netdev); |
1982 | netif_tx_stop_all_queues(dev: netdev); |
1983 | |
1984 | pf->flags |= OTX2_FLAG_INTF_DOWN; |
1985 | /* 'intf_down' may be checked on any cpu */ |
1986 | smp_wmb(); |
1987 | |
1988 | /* First stop packet Rx/Tx */ |
1989 | otx2_rxtx_enable(pfvf: pf, enable: false); |
1990 | |
1991 | /* Clear RSS enable flag */ |
1992 | rss = &pf->hw.rss_info; |
1993 | rss->enable = false; |
1994 | if (!netif_is_rxfh_configured(dev: netdev)) |
1995 | kfree(objp: rss->rss_ctx[DEFAULT_RSS_CONTEXT_GROUP]); |
1996 | |
1997 | /* Cleanup Queue IRQ */ |
1998 | vec = pci_irq_vector(dev: pf->pdev, |
1999 | nr: pf->hw.nix_msixoff + NIX_LF_QINT_VEC_START); |
2000 | otx2_write64(nic: pf, NIX_LF_QINTX_ENA_W1C(0), BIT_ULL(0)); |
2001 | free_irq(vec, pf); |
2002 | |
2003 | /* Cleanup CQ NAPI and IRQ */ |
2004 | vec = pf->hw.nix_msixoff + NIX_LF_CINT_VEC_START; |
2005 | for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) { |
2006 | /* Disable interrupt */ |
2007 | otx2_write64(nic: pf, NIX_LF_CINTX_ENA_W1C(qidx), BIT_ULL(0)); |
2008 | |
2009 | synchronize_irq(irq: pci_irq_vector(dev: pf->pdev, nr: vec)); |
2010 | |
2011 | cq_poll = &qset->napi[qidx]; |
2012 | napi_synchronize(n: &cq_poll->napi); |
2013 | vec++; |
2014 | } |
2015 | |
2016 | netif_tx_disable(dev: netdev); |
2017 | |
2018 | for (wrk = 0; wrk < pf->qset.cq_cnt; wrk++) |
2019 | cancel_delayed_work_sync(dwork: &pf->refill_wrk[wrk].pool_refill_work); |
2020 | devm_kfree(dev: pf->dev, p: pf->refill_wrk); |
2021 | |
2022 | otx2_free_hw_resources(pf); |
2023 | otx2_free_cints(pfvf: pf, n: pf->hw.cint_cnt); |
2024 | otx2_disable_napi(pf); |
2025 | |
2026 | for (qidx = 0; qidx < netdev->num_tx_queues; qidx++) |
2027 | netdev_tx_reset_queue(q: netdev_get_tx_queue(dev: netdev, index: qidx)); |
2028 | |
2029 | |
2030 | kfree(objp: qset->sq); |
2031 | kfree(objp: qset->cq); |
2032 | kfree(objp: qset->rq); |
2033 | kfree(objp: qset->napi); |
2034 | /* Do not clear RQ/SQ ringsize settings */ |
2035 | memset_startat(qset, 0, sqe_cnt); |
2036 | return 0; |
2037 | } |
2038 | EXPORT_SYMBOL(otx2_stop); |
2039 | |
2040 | static netdev_tx_t otx2_xmit(struct sk_buff *skb, struct net_device *netdev) |
2041 | { |
2042 | struct otx2_nic *pf = netdev_priv(dev: netdev); |
2043 | int qidx = skb_get_queue_mapping(skb); |
2044 | struct otx2_snd_queue *sq; |
2045 | struct netdev_queue *txq; |
2046 | int sq_idx; |
2047 | |
2048 | /* XDP SQs are not mapped with TXQs |
2049 | * advance qid to derive correct sq mapped with QOS |
2050 | */ |
2051 | sq_idx = (qidx >= pf->hw.tx_queues) ? (qidx + pf->hw.xdp_queues) : qidx; |
2052 | |
2053 | /* Check for minimum and maximum packet length */ |
2054 | if (skb->len <= ETH_HLEN || |
2055 | (!skb_shinfo(skb)->gso_size && skb->len > pf->tx_max_pktlen)) { |
2056 | dev_kfree_skb(skb); |
2057 | return NETDEV_TX_OK; |
2058 | } |
2059 | |
2060 | sq = &pf->qset.sq[sq_idx]; |
2061 | txq = netdev_get_tx_queue(dev: netdev, index: qidx); |
2062 | |
2063 | if (!otx2_sq_append_skb(netdev, sq, skb, qidx)) { |
2064 | netif_tx_stop_queue(dev_queue: txq); |
2065 | |
2066 | /* Check again, incase SQBs got freed up */ |
2067 | smp_mb(); |
2068 | if (((sq->num_sqbs - *sq->aura_fc_addr) * sq->sqe_per_sqb) |
2069 | > sq->sqe_thresh) |
2070 | netif_tx_wake_queue(dev_queue: txq); |
2071 | |
2072 | return NETDEV_TX_BUSY; |
2073 | } |
2074 | |
2075 | return NETDEV_TX_OK; |
2076 | } |
2077 | |
2078 | static int otx2_qos_select_htb_queue(struct otx2_nic *pf, struct sk_buff *skb, |
2079 | u16 htb_maj_id) |
2080 | { |
2081 | u16 classid; |
2082 | |
2083 | if ((TC_H_MAJ(skb->priority) >> 16) == htb_maj_id) |
2084 | classid = TC_H_MIN(skb->priority); |
2085 | else |
2086 | classid = READ_ONCE(pf->qos.defcls); |
2087 | |
2088 | if (!classid) |
2089 | return 0; |
2090 | |
2091 | return otx2_get_txq_by_classid(pfvf: pf, classid); |
2092 | } |
2093 | |
2094 | u16 otx2_select_queue(struct net_device *netdev, struct sk_buff *skb, |
2095 | struct net_device *sb_dev) |
2096 | { |
2097 | struct otx2_nic *pf = netdev_priv(dev: netdev); |
2098 | bool qos_enabled; |
2099 | #ifdef CONFIG_DCB |
2100 | u8 vlan_prio; |
2101 | #endif |
2102 | int txq; |
2103 | |
2104 | qos_enabled = netdev->real_num_tx_queues > pf->hw.tx_queues; |
2105 | if (unlikely(qos_enabled)) { |
2106 | /* This smp_load_acquire() pairs with smp_store_release() in |
2107 | * otx2_qos_root_add() called from htb offload root creation |
2108 | */ |
2109 | u16 htb_maj_id = smp_load_acquire(&pf->qos.maj_id); |
2110 | |
2111 | if (unlikely(htb_maj_id)) { |
2112 | txq = otx2_qos_select_htb_queue(pf, skb, htb_maj_id); |
2113 | if (txq > 0) |
2114 | return txq; |
2115 | goto process_pfc; |
2116 | } |
2117 | } |
2118 | |
2119 | process_pfc: |
2120 | #ifdef CONFIG_DCB |
2121 | if (!skb_vlan_tag_present(skb)) |
2122 | goto pick_tx; |
2123 | |
2124 | vlan_prio = skb->vlan_tci >> 13; |
2125 | if ((vlan_prio > pf->hw.tx_queues - 1) || |
2126 | !pf->pfc_alloc_status[vlan_prio]) |
2127 | goto pick_tx; |
2128 | |
2129 | return vlan_prio; |
2130 | |
2131 | pick_tx: |
2132 | #endif |
2133 | txq = netdev_pick_tx(dev: netdev, skb, NULL); |
2134 | if (unlikely(qos_enabled)) |
2135 | return txq % pf->hw.tx_queues; |
2136 | |
2137 | return txq; |
2138 | } |
2139 | EXPORT_SYMBOL(otx2_select_queue); |
2140 | |
2141 | static netdev_features_t otx2_fix_features(struct net_device *dev, |
2142 | netdev_features_t features) |
2143 | { |
2144 | if (features & NETIF_F_HW_VLAN_CTAG_RX) |
2145 | features |= NETIF_F_HW_VLAN_STAG_RX; |
2146 | else |
2147 | features &= ~NETIF_F_HW_VLAN_STAG_RX; |
2148 | |
2149 | return features; |
2150 | } |
2151 | |
2152 | static void otx2_set_rx_mode(struct net_device *netdev) |
2153 | { |
2154 | struct otx2_nic *pf = netdev_priv(dev: netdev); |
2155 | |
2156 | queue_work(wq: pf->otx2_wq, work: &pf->rx_mode_work); |
2157 | } |
2158 | |
2159 | static void otx2_rx_mode_wrk_handler(struct work_struct *work) |
2160 | { |
2161 | struct otx2_nic *pf = container_of(work, struct otx2_nic, rx_mode_work); |
2162 | |
2163 | otx2_do_set_rx_mode(pf); |
2164 | } |
2165 | |
2166 | static int otx2_set_features(struct net_device *netdev, |
2167 | netdev_features_t features) |
2168 | { |
2169 | netdev_features_t changed = features ^ netdev->features; |
2170 | struct otx2_nic *pf = netdev_priv(dev: netdev); |
2171 | |
2172 | if ((changed & NETIF_F_LOOPBACK) && netif_running(dev: netdev)) |
2173 | return otx2_cgx_config_loopback(pf, |
2174 | enable: features & NETIF_F_LOOPBACK); |
2175 | |
2176 | if ((changed & NETIF_F_HW_VLAN_CTAG_RX) && netif_running(dev: netdev)) |
2177 | return otx2_enable_rxvlan(pf, |
2178 | enable: features & NETIF_F_HW_VLAN_CTAG_RX); |
2179 | |
2180 | return otx2_handle_ntuple_tc_features(netdev, features); |
2181 | } |
2182 | |
2183 | static void otx2_reset_task(struct work_struct *work) |
2184 | { |
2185 | struct otx2_nic *pf = container_of(work, struct otx2_nic, reset_task); |
2186 | |
2187 | if (!netif_running(dev: pf->netdev)) |
2188 | return; |
2189 | |
2190 | rtnl_lock(); |
2191 | otx2_stop(pf->netdev); |
2192 | pf->reset_count++; |
2193 | otx2_open(pf->netdev); |
2194 | netif_trans_update(dev: pf->netdev); |
2195 | rtnl_unlock(); |
2196 | } |
2197 | |
2198 | static int otx2_config_hw_rx_tstamp(struct otx2_nic *pfvf, bool enable) |
2199 | { |
2200 | struct msg_req *req; |
2201 | int err; |
2202 | |
2203 | if (pfvf->flags & OTX2_FLAG_RX_TSTAMP_ENABLED && enable) |
2204 | return 0; |
2205 | |
2206 | mutex_lock(&pfvf->mbox.lock); |
2207 | if (enable) |
2208 | req = otx2_mbox_alloc_msg_cgx_ptp_rx_enable(mbox: &pfvf->mbox); |
2209 | else |
2210 | req = otx2_mbox_alloc_msg_cgx_ptp_rx_disable(mbox: &pfvf->mbox); |
2211 | if (!req) { |
2212 | mutex_unlock(lock: &pfvf->mbox.lock); |
2213 | return -ENOMEM; |
2214 | } |
2215 | |
2216 | err = otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
2217 | if (err) { |
2218 | mutex_unlock(lock: &pfvf->mbox.lock); |
2219 | return err; |
2220 | } |
2221 | |
2222 | mutex_unlock(lock: &pfvf->mbox.lock); |
2223 | if (enable) |
2224 | pfvf->flags |= OTX2_FLAG_RX_TSTAMP_ENABLED; |
2225 | else |
2226 | pfvf->flags &= ~OTX2_FLAG_RX_TSTAMP_ENABLED; |
2227 | return 0; |
2228 | } |
2229 | |
2230 | static int otx2_config_hw_tx_tstamp(struct otx2_nic *pfvf, bool enable) |
2231 | { |
2232 | struct msg_req *req; |
2233 | int err; |
2234 | |
2235 | if (pfvf->flags & OTX2_FLAG_TX_TSTAMP_ENABLED && enable) |
2236 | return 0; |
2237 | |
2238 | mutex_lock(&pfvf->mbox.lock); |
2239 | if (enable) |
2240 | req = otx2_mbox_alloc_msg_nix_lf_ptp_tx_enable(mbox: &pfvf->mbox); |
2241 | else |
2242 | req = otx2_mbox_alloc_msg_nix_lf_ptp_tx_disable(mbox: &pfvf->mbox); |
2243 | if (!req) { |
2244 | mutex_unlock(lock: &pfvf->mbox.lock); |
2245 | return -ENOMEM; |
2246 | } |
2247 | |
2248 | err = otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
2249 | if (err) { |
2250 | mutex_unlock(lock: &pfvf->mbox.lock); |
2251 | return err; |
2252 | } |
2253 | |
2254 | mutex_unlock(lock: &pfvf->mbox.lock); |
2255 | if (enable) |
2256 | pfvf->flags |= OTX2_FLAG_TX_TSTAMP_ENABLED; |
2257 | else |
2258 | pfvf->flags &= ~OTX2_FLAG_TX_TSTAMP_ENABLED; |
2259 | return 0; |
2260 | } |
2261 | |
2262 | int otx2_config_hwtstamp(struct net_device *netdev, struct ifreq *ifr) |
2263 | { |
2264 | struct otx2_nic *pfvf = netdev_priv(dev: netdev); |
2265 | struct hwtstamp_config config; |
2266 | |
2267 | if (!pfvf->ptp) |
2268 | return -ENODEV; |
2269 | |
2270 | if (copy_from_user(to: &config, from: ifr->ifr_data, n: sizeof(config))) |
2271 | return -EFAULT; |
2272 | |
2273 | switch (config.tx_type) { |
2274 | case HWTSTAMP_TX_OFF: |
2275 | if (pfvf->flags & OTX2_FLAG_PTP_ONESTEP_SYNC) |
2276 | pfvf->flags &= ~OTX2_FLAG_PTP_ONESTEP_SYNC; |
2277 | |
2278 | cancel_delayed_work(dwork: &pfvf->ptp->synctstamp_work); |
2279 | otx2_config_hw_tx_tstamp(pfvf, enable: false); |
2280 | break; |
2281 | case HWTSTAMP_TX_ONESTEP_SYNC: |
2282 | if (!test_bit(CN10K_PTP_ONESTEP, &pfvf->hw.cap_flag)) |
2283 | return -ERANGE; |
2284 | pfvf->flags |= OTX2_FLAG_PTP_ONESTEP_SYNC; |
2285 | schedule_delayed_work(dwork: &pfvf->ptp->synctstamp_work, |
2286 | delay: msecs_to_jiffies(m: 500)); |
2287 | fallthrough; |
2288 | case HWTSTAMP_TX_ON: |
2289 | otx2_config_hw_tx_tstamp(pfvf, enable: true); |
2290 | break; |
2291 | default: |
2292 | return -ERANGE; |
2293 | } |
2294 | |
2295 | switch (config.rx_filter) { |
2296 | case HWTSTAMP_FILTER_NONE: |
2297 | otx2_config_hw_rx_tstamp(pfvf, enable: false); |
2298 | break; |
2299 | case HWTSTAMP_FILTER_ALL: |
2300 | case HWTSTAMP_FILTER_SOME: |
2301 | case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: |
2302 | case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: |
2303 | case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: |
2304 | case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: |
2305 | case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: |
2306 | case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: |
2307 | case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: |
2308 | case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: |
2309 | case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: |
2310 | case HWTSTAMP_FILTER_PTP_V2_EVENT: |
2311 | case HWTSTAMP_FILTER_PTP_V2_SYNC: |
2312 | case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: |
2313 | otx2_config_hw_rx_tstamp(pfvf, enable: true); |
2314 | config.rx_filter = HWTSTAMP_FILTER_ALL; |
2315 | break; |
2316 | default: |
2317 | return -ERANGE; |
2318 | } |
2319 | |
2320 | memcpy(&pfvf->tstamp, &config, sizeof(config)); |
2321 | |
2322 | return copy_to_user(to: ifr->ifr_data, from: &config, |
2323 | n: sizeof(config)) ? -EFAULT : 0; |
2324 | } |
2325 | EXPORT_SYMBOL(otx2_config_hwtstamp); |
2326 | |
2327 | int otx2_ioctl(struct net_device *netdev, struct ifreq *req, int cmd) |
2328 | { |
2329 | struct otx2_nic *pfvf = netdev_priv(dev: netdev); |
2330 | struct hwtstamp_config *cfg = &pfvf->tstamp; |
2331 | |
2332 | switch (cmd) { |
2333 | case SIOCSHWTSTAMP: |
2334 | return otx2_config_hwtstamp(netdev, req); |
2335 | case SIOCGHWTSTAMP: |
2336 | return copy_to_user(to: req->ifr_data, from: cfg, |
2337 | n: sizeof(*cfg)) ? -EFAULT : 0; |
2338 | default: |
2339 | return -EOPNOTSUPP; |
2340 | } |
2341 | } |
2342 | EXPORT_SYMBOL(otx2_ioctl); |
2343 | |
2344 | static int otx2_do_set_vf_mac(struct otx2_nic *pf, int vf, const u8 *mac) |
2345 | { |
2346 | struct npc_install_flow_req *req; |
2347 | int err; |
2348 | |
2349 | mutex_lock(&pf->mbox.lock); |
2350 | req = otx2_mbox_alloc_msg_npc_install_flow(mbox: &pf->mbox); |
2351 | if (!req) { |
2352 | err = -ENOMEM; |
2353 | goto out; |
2354 | } |
2355 | |
2356 | ether_addr_copy(dst: req->packet.dmac, src: mac); |
2357 | eth_broadcast_addr(addr: (u8 *)&req->mask.dmac); |
2358 | req->features = BIT_ULL(NPC_DMAC); |
2359 | req->channel = pf->hw.rx_chan_base; |
2360 | req->intf = NIX_INTF_RX; |
2361 | req->default_rule = 1; |
2362 | req->append = 1; |
2363 | req->vf = vf + 1; |
2364 | req->op = NIX_RX_ACTION_DEFAULT; |
2365 | |
2366 | err = otx2_sync_mbox_msg(mbox: &pf->mbox); |
2367 | out: |
2368 | mutex_unlock(lock: &pf->mbox.lock); |
2369 | return err; |
2370 | } |
2371 | |
2372 | static int otx2_set_vf_mac(struct net_device *netdev, int vf, u8 *mac) |
2373 | { |
2374 | struct otx2_nic *pf = netdev_priv(dev: netdev); |
2375 | struct pci_dev *pdev = pf->pdev; |
2376 | struct otx2_vf_config *config; |
2377 | int ret; |
2378 | |
2379 | if (!netif_running(dev: netdev)) |
2380 | return -EAGAIN; |
2381 | |
2382 | if (vf >= pf->total_vfs) |
2383 | return -EINVAL; |
2384 | |
2385 | if (!is_valid_ether_addr(addr: mac)) |
2386 | return -EINVAL; |
2387 | |
2388 | config = &pf->vf_configs[vf]; |
2389 | ether_addr_copy(dst: config->mac, src: mac); |
2390 | |
2391 | ret = otx2_do_set_vf_mac(pf, vf, mac); |
2392 | if (ret == 0) |
2393 | dev_info(&pdev->dev, |
2394 | "Load/Reload VF driver\n" ); |
2395 | |
2396 | return ret; |
2397 | } |
2398 | |
2399 | static int otx2_do_set_vf_vlan(struct otx2_nic *pf, int vf, u16 vlan, u8 qos, |
2400 | __be16 proto) |
2401 | { |
2402 | struct otx2_flow_config *flow_cfg = pf->flow_cfg; |
2403 | struct nix_vtag_config_rsp *vtag_rsp; |
2404 | struct npc_delete_flow_req *del_req; |
2405 | struct nix_vtag_config *vtag_req; |
2406 | struct npc_install_flow_req *req; |
2407 | struct otx2_vf_config *config; |
2408 | int err = 0; |
2409 | u32 idx; |
2410 | |
2411 | config = &pf->vf_configs[vf]; |
2412 | |
2413 | if (!vlan && !config->vlan) |
2414 | goto out; |
2415 | |
2416 | mutex_lock(&pf->mbox.lock); |
2417 | |
2418 | /* free old tx vtag entry */ |
2419 | if (config->vlan) { |
2420 | vtag_req = otx2_mbox_alloc_msg_nix_vtag_cfg(mbox: &pf->mbox); |
2421 | if (!vtag_req) { |
2422 | err = -ENOMEM; |
2423 | goto out; |
2424 | } |
2425 | vtag_req->cfg_type = 0; |
2426 | vtag_req->tx.free_vtag0 = 1; |
2427 | vtag_req->tx.vtag0_idx = config->tx_vtag_idx; |
2428 | |
2429 | err = otx2_sync_mbox_msg(mbox: &pf->mbox); |
2430 | if (err) |
2431 | goto out; |
2432 | } |
2433 | |
2434 | if (!vlan && config->vlan) { |
2435 | /* rx */ |
2436 | del_req = otx2_mbox_alloc_msg_npc_delete_flow(mbox: &pf->mbox); |
2437 | if (!del_req) { |
2438 | err = -ENOMEM; |
2439 | goto out; |
2440 | } |
2441 | idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_RX_INDEX); |
2442 | del_req->entry = |
2443 | flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx]; |
2444 | err = otx2_sync_mbox_msg(mbox: &pf->mbox); |
2445 | if (err) |
2446 | goto out; |
2447 | |
2448 | /* tx */ |
2449 | del_req = otx2_mbox_alloc_msg_npc_delete_flow(mbox: &pf->mbox); |
2450 | if (!del_req) { |
2451 | err = -ENOMEM; |
2452 | goto out; |
2453 | } |
2454 | idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_TX_INDEX); |
2455 | del_req->entry = |
2456 | flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx]; |
2457 | err = otx2_sync_mbox_msg(mbox: &pf->mbox); |
2458 | |
2459 | goto out; |
2460 | } |
2461 | |
2462 | /* rx */ |
2463 | req = otx2_mbox_alloc_msg_npc_install_flow(mbox: &pf->mbox); |
2464 | if (!req) { |
2465 | err = -ENOMEM; |
2466 | goto out; |
2467 | } |
2468 | |
2469 | idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_RX_INDEX); |
2470 | req->entry = flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx]; |
2471 | req->packet.vlan_tci = htons(vlan); |
2472 | req->mask.vlan_tci = htons(VLAN_VID_MASK); |
2473 | /* af fills the destination mac addr */ |
2474 | eth_broadcast_addr(addr: (u8 *)&req->mask.dmac); |
2475 | req->features = BIT_ULL(NPC_OUTER_VID) | BIT_ULL(NPC_DMAC); |
2476 | req->channel = pf->hw.rx_chan_base; |
2477 | req->intf = NIX_INTF_RX; |
2478 | req->vf = vf + 1; |
2479 | req->op = NIX_RX_ACTION_DEFAULT; |
2480 | req->vtag0_valid = true; |
2481 | req->vtag0_type = NIX_AF_LFX_RX_VTAG_TYPE7; |
2482 | req->set_cntr = 1; |
2483 | |
2484 | err = otx2_sync_mbox_msg(mbox: &pf->mbox); |
2485 | if (err) |
2486 | goto out; |
2487 | |
2488 | /* tx */ |
2489 | vtag_req = otx2_mbox_alloc_msg_nix_vtag_cfg(mbox: &pf->mbox); |
2490 | if (!vtag_req) { |
2491 | err = -ENOMEM; |
2492 | goto out; |
2493 | } |
2494 | |
2495 | /* configure tx vtag params */ |
2496 | vtag_req->vtag_size = VTAGSIZE_T4; |
2497 | vtag_req->cfg_type = 0; /* tx vlan cfg */ |
2498 | vtag_req->tx.cfg_vtag0 = 1; |
2499 | vtag_req->tx.vtag0 = ((u64)ntohs(proto) << 16) | vlan; |
2500 | |
2501 | err = otx2_sync_mbox_msg(mbox: &pf->mbox); |
2502 | if (err) |
2503 | goto out; |
2504 | |
2505 | vtag_rsp = (struct nix_vtag_config_rsp *)otx2_mbox_get_rsp |
2506 | (mbox: &pf->mbox.mbox, devid: 0, msg: &vtag_req->hdr); |
2507 | if (IS_ERR(ptr: vtag_rsp)) { |
2508 | err = PTR_ERR(ptr: vtag_rsp); |
2509 | goto out; |
2510 | } |
2511 | config->tx_vtag_idx = vtag_rsp->vtag0_idx; |
2512 | |
2513 | req = otx2_mbox_alloc_msg_npc_install_flow(mbox: &pf->mbox); |
2514 | if (!req) { |
2515 | err = -ENOMEM; |
2516 | goto out; |
2517 | } |
2518 | |
2519 | eth_zero_addr(addr: (u8 *)&req->mask.dmac); |
2520 | idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_TX_INDEX); |
2521 | req->entry = flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx]; |
2522 | req->features = BIT_ULL(NPC_DMAC); |
2523 | req->channel = pf->hw.tx_chan_base; |
2524 | req->intf = NIX_INTF_TX; |
2525 | req->vf = vf + 1; |
2526 | req->op = NIX_TX_ACTIONOP_UCAST_DEFAULT; |
2527 | req->vtag0_def = vtag_rsp->vtag0_idx; |
2528 | req->vtag0_op = VTAG_INSERT; |
2529 | req->set_cntr = 1; |
2530 | |
2531 | err = otx2_sync_mbox_msg(mbox: &pf->mbox); |
2532 | out: |
2533 | config->vlan = vlan; |
2534 | mutex_unlock(lock: &pf->mbox.lock); |
2535 | return err; |
2536 | } |
2537 | |
2538 | static int otx2_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos, |
2539 | __be16 proto) |
2540 | { |
2541 | struct otx2_nic *pf = netdev_priv(dev: netdev); |
2542 | struct pci_dev *pdev = pf->pdev; |
2543 | |
2544 | if (!netif_running(dev: netdev)) |
2545 | return -EAGAIN; |
2546 | |
2547 | if (vf >= pci_num_vf(dev: pdev)) |
2548 | return -EINVAL; |
2549 | |
2550 | /* qos is currently unsupported */ |
2551 | if (vlan >= VLAN_N_VID || qos) |
2552 | return -EINVAL; |
2553 | |
2554 | if (proto != htons(ETH_P_8021Q)) |
2555 | return -EPROTONOSUPPORT; |
2556 | |
2557 | if (!(pf->flags & OTX2_FLAG_VF_VLAN_SUPPORT)) |
2558 | return -EOPNOTSUPP; |
2559 | |
2560 | return otx2_do_set_vf_vlan(pf, vf, vlan, qos, proto); |
2561 | } |
2562 | |
2563 | static int otx2_get_vf_config(struct net_device *netdev, int vf, |
2564 | struct ifla_vf_info *ivi) |
2565 | { |
2566 | struct otx2_nic *pf = netdev_priv(dev: netdev); |
2567 | struct pci_dev *pdev = pf->pdev; |
2568 | struct otx2_vf_config *config; |
2569 | |
2570 | if (!netif_running(dev: netdev)) |
2571 | return -EAGAIN; |
2572 | |
2573 | if (vf >= pci_num_vf(dev: pdev)) |
2574 | return -EINVAL; |
2575 | |
2576 | config = &pf->vf_configs[vf]; |
2577 | ivi->vf = vf; |
2578 | ether_addr_copy(dst: ivi->mac, src: config->mac); |
2579 | ivi->vlan = config->vlan; |
2580 | ivi->trusted = config->trusted; |
2581 | |
2582 | return 0; |
2583 | } |
2584 | |
2585 | static int otx2_xdp_xmit_tx(struct otx2_nic *pf, struct xdp_frame *xdpf, |
2586 | int qidx) |
2587 | { |
2588 | struct page *page; |
2589 | u64 dma_addr; |
2590 | int err = 0; |
2591 | |
2592 | dma_addr = otx2_dma_map_page(pfvf: pf, virt_to_page(xdpf->data), |
2593 | offset_in_page(xdpf->data), size: xdpf->len, |
2594 | dir: DMA_TO_DEVICE); |
2595 | if (dma_mapping_error(dev: pf->dev, dma_addr)) |
2596 | return -ENOMEM; |
2597 | |
2598 | err = otx2_xdp_sq_append_pkt(pfvf: pf, iova: dma_addr, len: xdpf->len, qidx); |
2599 | if (!err) { |
2600 | otx2_dma_unmap_page(pfvf: pf, addr: dma_addr, size: xdpf->len, dir: DMA_TO_DEVICE); |
2601 | page = virt_to_page(xdpf->data); |
2602 | put_page(page); |
2603 | return -ENOMEM; |
2604 | } |
2605 | return 0; |
2606 | } |
2607 | |
2608 | static int otx2_xdp_xmit(struct net_device *netdev, int n, |
2609 | struct xdp_frame **frames, u32 flags) |
2610 | { |
2611 | struct otx2_nic *pf = netdev_priv(dev: netdev); |
2612 | int qidx = smp_processor_id(); |
2613 | struct otx2_snd_queue *sq; |
2614 | int drops = 0, i; |
2615 | |
2616 | if (!netif_running(dev: netdev)) |
2617 | return -ENETDOWN; |
2618 | |
2619 | qidx += pf->hw.tx_queues; |
2620 | sq = pf->xdp_prog ? &pf->qset.sq[qidx] : NULL; |
2621 | |
2622 | /* Abort xmit if xdp queue is not */ |
2623 | if (unlikely(!sq)) |
2624 | return -ENXIO; |
2625 | |
2626 | if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) |
2627 | return -EINVAL; |
2628 | |
2629 | for (i = 0; i < n; i++) { |
2630 | struct xdp_frame *xdpf = frames[i]; |
2631 | int err; |
2632 | |
2633 | err = otx2_xdp_xmit_tx(pf, xdpf, qidx); |
2634 | if (err) |
2635 | drops++; |
2636 | } |
2637 | return n - drops; |
2638 | } |
2639 | |
2640 | static int otx2_xdp_setup(struct otx2_nic *pf, struct bpf_prog *prog) |
2641 | { |
2642 | struct net_device *dev = pf->netdev; |
2643 | bool if_up = netif_running(dev: pf->netdev); |
2644 | struct bpf_prog *old_prog; |
2645 | |
2646 | if (prog && dev->mtu > MAX_XDP_MTU) { |
2647 | netdev_warn(dev, format: "Jumbo frames not yet supported with XDP\n" ); |
2648 | return -EOPNOTSUPP; |
2649 | } |
2650 | |
2651 | if (if_up) |
2652 | otx2_stop(pf->netdev); |
2653 | |
2654 | old_prog = xchg(&pf->xdp_prog, prog); |
2655 | |
2656 | if (old_prog) |
2657 | bpf_prog_put(prog: old_prog); |
2658 | |
2659 | if (pf->xdp_prog) |
2660 | bpf_prog_add(prog: pf->xdp_prog, i: pf->hw.rx_queues - 1); |
2661 | |
2662 | /* Network stack and XDP shared same rx queues. |
2663 | * Use separate tx queues for XDP and network stack. |
2664 | */ |
2665 | if (pf->xdp_prog) { |
2666 | pf->hw.xdp_queues = pf->hw.rx_queues; |
2667 | xdp_features_set_redirect_target(dev, support_sg: false); |
2668 | } else { |
2669 | pf->hw.xdp_queues = 0; |
2670 | xdp_features_clear_redirect_target(dev); |
2671 | } |
2672 | |
2673 | if (if_up) |
2674 | otx2_open(pf->netdev); |
2675 | |
2676 | return 0; |
2677 | } |
2678 | |
2679 | static int otx2_xdp(struct net_device *netdev, struct netdev_bpf *xdp) |
2680 | { |
2681 | struct otx2_nic *pf = netdev_priv(dev: netdev); |
2682 | |
2683 | switch (xdp->command) { |
2684 | case XDP_SETUP_PROG: |
2685 | return otx2_xdp_setup(pf, prog: xdp->prog); |
2686 | default: |
2687 | return -EINVAL; |
2688 | } |
2689 | } |
2690 | |
2691 | static int otx2_set_vf_permissions(struct otx2_nic *pf, int vf, |
2692 | int req_perm) |
2693 | { |
2694 | struct set_vf_perm *req; |
2695 | int rc; |
2696 | |
2697 | mutex_lock(&pf->mbox.lock); |
2698 | req = otx2_mbox_alloc_msg_set_vf_perm(mbox: &pf->mbox); |
2699 | if (!req) { |
2700 | rc = -ENOMEM; |
2701 | goto out; |
2702 | } |
2703 | |
2704 | /* Let AF reset VF permissions as sriov is disabled */ |
2705 | if (req_perm == OTX2_RESET_VF_PERM) { |
2706 | req->flags |= RESET_VF_PERM; |
2707 | } else if (req_perm == OTX2_TRUSTED_VF) { |
2708 | if (pf->vf_configs[vf].trusted) |
2709 | req->flags |= VF_TRUSTED; |
2710 | } |
2711 | |
2712 | req->vf = vf; |
2713 | rc = otx2_sync_mbox_msg(mbox: &pf->mbox); |
2714 | out: |
2715 | mutex_unlock(lock: &pf->mbox.lock); |
2716 | return rc; |
2717 | } |
2718 | |
2719 | static int otx2_ndo_set_vf_trust(struct net_device *netdev, int vf, |
2720 | bool enable) |
2721 | { |
2722 | struct otx2_nic *pf = netdev_priv(dev: netdev); |
2723 | struct pci_dev *pdev = pf->pdev; |
2724 | int rc; |
2725 | |
2726 | if (vf >= pci_num_vf(dev: pdev)) |
2727 | return -EINVAL; |
2728 | |
2729 | if (pf->vf_configs[vf].trusted == enable) |
2730 | return 0; |
2731 | |
2732 | pf->vf_configs[vf].trusted = enable; |
2733 | rc = otx2_set_vf_permissions(pf, vf, req_perm: OTX2_TRUSTED_VF); |
2734 | |
2735 | if (rc) { |
2736 | pf->vf_configs[vf].trusted = !enable; |
2737 | } else { |
2738 | netdev_info(dev: pf->netdev, format: "VF %d is %strusted\n" , |
2739 | vf, enable ? "" : "not " ); |
2740 | otx2_set_rx_mode(netdev); |
2741 | } |
2742 | |
2743 | return rc; |
2744 | } |
2745 | |
2746 | static const struct net_device_ops otx2_netdev_ops = { |
2747 | .ndo_open = otx2_open, |
2748 | .ndo_stop = otx2_stop, |
2749 | .ndo_start_xmit = otx2_xmit, |
2750 | .ndo_select_queue = otx2_select_queue, |
2751 | .ndo_fix_features = otx2_fix_features, |
2752 | .ndo_set_mac_address = otx2_set_mac_address, |
2753 | .ndo_change_mtu = otx2_change_mtu, |
2754 | .ndo_set_rx_mode = otx2_set_rx_mode, |
2755 | .ndo_set_features = otx2_set_features, |
2756 | .ndo_tx_timeout = otx2_tx_timeout, |
2757 | .ndo_get_stats64 = otx2_get_stats64, |
2758 | .ndo_eth_ioctl = otx2_ioctl, |
2759 | .ndo_set_vf_mac = otx2_set_vf_mac, |
2760 | .ndo_set_vf_vlan = otx2_set_vf_vlan, |
2761 | .ndo_get_vf_config = otx2_get_vf_config, |
2762 | .ndo_bpf = otx2_xdp, |
2763 | .ndo_xdp_xmit = otx2_xdp_xmit, |
2764 | .ndo_setup_tc = otx2_setup_tc, |
2765 | .ndo_set_vf_trust = otx2_ndo_set_vf_trust, |
2766 | }; |
2767 | |
2768 | static int otx2_wq_init(struct otx2_nic *pf) |
2769 | { |
2770 | pf->otx2_wq = create_singlethread_workqueue("otx2_wq" ); |
2771 | if (!pf->otx2_wq) |
2772 | return -ENOMEM; |
2773 | |
2774 | INIT_WORK(&pf->rx_mode_work, otx2_rx_mode_wrk_handler); |
2775 | INIT_WORK(&pf->reset_task, otx2_reset_task); |
2776 | return 0; |
2777 | } |
2778 | |
2779 | static int otx2_check_pf_usable(struct otx2_nic *nic) |
2780 | { |
2781 | u64 rev; |
2782 | |
2783 | rev = otx2_read64(nic, RVU_PF_BLOCK_ADDRX_DISC(BLKADDR_RVUM)); |
2784 | rev = (rev >> 12) & 0xFF; |
2785 | /* Check if AF has setup revision for RVUM block, |
2786 | * otherwise this driver probe should be deferred |
2787 | * until AF driver comes up. |
2788 | */ |
2789 | if (!rev) { |
2790 | dev_warn(nic->dev, |
2791 | "AF is not initialized, deferring probe\n" ); |
2792 | return -EPROBE_DEFER; |
2793 | } |
2794 | return 0; |
2795 | } |
2796 | |
2797 | static int otx2_realloc_msix_vectors(struct otx2_nic *pf) |
2798 | { |
2799 | struct otx2_hw *hw = &pf->hw; |
2800 | int num_vec, err; |
2801 | |
2802 | /* NPA interrupts are inot registered, so alloc only |
2803 | * upto NIX vector offset. |
2804 | */ |
2805 | num_vec = hw->nix_msixoff; |
2806 | num_vec += NIX_LF_CINT_VEC_START + hw->max_queues; |
2807 | |
2808 | otx2_disable_mbox_intr(pf); |
2809 | pci_free_irq_vectors(dev: hw->pdev); |
2810 | err = pci_alloc_irq_vectors(dev: hw->pdev, min_vecs: num_vec, max_vecs: num_vec, PCI_IRQ_MSIX); |
2811 | if (err < 0) { |
2812 | dev_err(pf->dev, "%s: Failed to realloc %d IRQ vectors\n" , |
2813 | __func__, num_vec); |
2814 | return err; |
2815 | } |
2816 | |
2817 | return otx2_register_mbox_intr(pf, probe_af: false); |
2818 | } |
2819 | |
2820 | static int otx2_sriov_vfcfg_init(struct otx2_nic *pf) |
2821 | { |
2822 | int i; |
2823 | |
2824 | pf->vf_configs = devm_kcalloc(dev: pf->dev, n: pf->total_vfs, |
2825 | size: sizeof(struct otx2_vf_config), |
2826 | GFP_KERNEL); |
2827 | if (!pf->vf_configs) |
2828 | return -ENOMEM; |
2829 | |
2830 | for (i = 0; i < pf->total_vfs; i++) { |
2831 | pf->vf_configs[i].pf = pf; |
2832 | pf->vf_configs[i].intf_down = true; |
2833 | pf->vf_configs[i].trusted = false; |
2834 | INIT_DELAYED_WORK(&pf->vf_configs[i].link_event_work, |
2835 | otx2_vf_link_event_task); |
2836 | } |
2837 | |
2838 | return 0; |
2839 | } |
2840 | |
2841 | static void otx2_sriov_vfcfg_cleanup(struct otx2_nic *pf) |
2842 | { |
2843 | int i; |
2844 | |
2845 | if (!pf->vf_configs) |
2846 | return; |
2847 | |
2848 | for (i = 0; i < pf->total_vfs; i++) { |
2849 | cancel_delayed_work_sync(dwork: &pf->vf_configs[i].link_event_work); |
2850 | otx2_set_vf_permissions(pf, vf: i, req_perm: OTX2_RESET_VF_PERM); |
2851 | } |
2852 | } |
2853 | |
2854 | static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id) |
2855 | { |
2856 | struct device *dev = &pdev->dev; |
2857 | int err, qcount, qos_txqs; |
2858 | struct net_device *netdev; |
2859 | struct otx2_nic *pf; |
2860 | struct otx2_hw *hw; |
2861 | int num_vec; |
2862 | |
2863 | err = pcim_enable_device(pdev); |
2864 | if (err) { |
2865 | dev_err(dev, "Failed to enable PCI device\n" ); |
2866 | return err; |
2867 | } |
2868 | |
2869 | err = pci_request_regions(pdev, DRV_NAME); |
2870 | if (err) { |
2871 | dev_err(dev, "PCI request regions failed 0x%x\n" , err); |
2872 | return err; |
2873 | } |
2874 | |
2875 | err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); |
2876 | if (err) { |
2877 | dev_err(dev, "DMA mask config failed, abort\n" ); |
2878 | goto err_release_regions; |
2879 | } |
2880 | |
2881 | pci_set_master(dev: pdev); |
2882 | |
2883 | /* Set number of queues */ |
2884 | qcount = min_t(int, num_online_cpus(), OTX2_MAX_CQ_CNT); |
2885 | qos_txqs = min_t(int, qcount, OTX2_QOS_MAX_LEAF_NODES); |
2886 | |
2887 | netdev = alloc_etherdev_mqs(sizeof_priv: sizeof(*pf), txqs: qcount + qos_txqs, rxqs: qcount); |
2888 | if (!netdev) { |
2889 | err = -ENOMEM; |
2890 | goto err_release_regions; |
2891 | } |
2892 | |
2893 | pci_set_drvdata(pdev, data: netdev); |
2894 | SET_NETDEV_DEV(netdev, &pdev->dev); |
2895 | pf = netdev_priv(dev: netdev); |
2896 | pf->netdev = netdev; |
2897 | pf->pdev = pdev; |
2898 | pf->dev = dev; |
2899 | pf->total_vfs = pci_sriov_get_totalvfs(dev: pdev); |
2900 | pf->flags |= OTX2_FLAG_INTF_DOWN; |
2901 | |
2902 | hw = &pf->hw; |
2903 | hw->pdev = pdev; |
2904 | hw->rx_queues = qcount; |
2905 | hw->tx_queues = qcount; |
2906 | hw->non_qos_queues = qcount; |
2907 | hw->max_queues = qcount; |
2908 | hw->rbuf_len = OTX2_DEFAULT_RBUF_LEN; |
2909 | /* Use CQE of 128 byte descriptor size by default */ |
2910 | hw->xqe_size = 128; |
2911 | |
2912 | num_vec = pci_msix_vec_count(dev: pdev); |
2913 | hw->irq_name = devm_kmalloc_array(dev: &hw->pdev->dev, n: num_vec, NAME_SIZE, |
2914 | GFP_KERNEL); |
2915 | if (!hw->irq_name) { |
2916 | err = -ENOMEM; |
2917 | goto err_free_netdev; |
2918 | } |
2919 | |
2920 | hw->affinity_mask = devm_kcalloc(dev: &hw->pdev->dev, n: num_vec, |
2921 | size: sizeof(cpumask_var_t), GFP_KERNEL); |
2922 | if (!hw->affinity_mask) { |
2923 | err = -ENOMEM; |
2924 | goto err_free_netdev; |
2925 | } |
2926 | |
2927 | /* Map CSRs */ |
2928 | pf->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, maxlen: 0); |
2929 | if (!pf->reg_base) { |
2930 | dev_err(dev, "Unable to map physical function CSRs, aborting\n" ); |
2931 | err = -ENOMEM; |
2932 | goto err_free_netdev; |
2933 | } |
2934 | |
2935 | err = otx2_check_pf_usable(nic: pf); |
2936 | if (err) |
2937 | goto err_free_netdev; |
2938 | |
2939 | err = pci_alloc_irq_vectors(dev: hw->pdev, min_vecs: RVU_PF_INT_VEC_CNT, |
2940 | max_vecs: RVU_PF_INT_VEC_CNT, PCI_IRQ_MSIX); |
2941 | if (err < 0) { |
2942 | dev_err(dev, "%s: Failed to alloc %d IRQ vectors\n" , |
2943 | __func__, num_vec); |
2944 | goto err_free_netdev; |
2945 | } |
2946 | |
2947 | otx2_setup_dev_hw_settings(pfvf: pf); |
2948 | |
2949 | /* Init PF <=> AF mailbox stuff */ |
2950 | err = otx2_pfaf_mbox_init(pf); |
2951 | if (err) |
2952 | goto err_free_irq_vectors; |
2953 | |
2954 | /* Register mailbox interrupt */ |
2955 | err = otx2_register_mbox_intr(pf, probe_af: true); |
2956 | if (err) |
2957 | goto err_mbox_destroy; |
2958 | |
2959 | /* Request AF to attach NPA and NIX LFs to this PF. |
2960 | * NIX and NPA LFs are needed for this PF to function as a NIC. |
2961 | */ |
2962 | err = otx2_attach_npa_nix(pfvf: pf); |
2963 | if (err) |
2964 | goto err_disable_mbox_intr; |
2965 | |
2966 | err = otx2_realloc_msix_vectors(pf); |
2967 | if (err) |
2968 | goto err_detach_rsrc; |
2969 | |
2970 | err = otx2_set_real_num_queues(netdev, hw->tx_queues, hw->rx_queues); |
2971 | if (err) |
2972 | goto err_detach_rsrc; |
2973 | |
2974 | err = cn10k_lmtst_init(pfvf: pf); |
2975 | if (err) |
2976 | goto err_detach_rsrc; |
2977 | |
2978 | /* Assign default mac address */ |
2979 | otx2_get_mac_from_af(netdev); |
2980 | |
2981 | /* Don't check for error. Proceed without ptp */ |
2982 | otx2_ptp_init(pfvf: pf); |
2983 | |
2984 | /* NPA's pool is a stack to which SW frees buffer pointers via Aura. |
2985 | * HW allocates buffer pointer from stack and uses it for DMA'ing |
2986 | * ingress packet. In some scenarios HW can free back allocated buffer |
2987 | * pointers to pool. This makes it impossible for SW to maintain a |
2988 | * parallel list where physical addresses of buffer pointers (IOVAs) |
2989 | * given to HW can be saved for later reference. |
2990 | * |
2991 | * So the only way to convert Rx packet's buffer address is to use |
2992 | * IOMMU's iova_to_phys() handler which translates the address by |
2993 | * walking through the translation tables. |
2994 | */ |
2995 | pf->iommu_domain = iommu_get_domain_for_dev(dev); |
2996 | |
2997 | netdev->hw_features = (NETIF_F_RXCSUM | NETIF_F_IP_CSUM | |
2998 | NETIF_F_IPV6_CSUM | NETIF_F_RXHASH | |
2999 | NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 | |
3000 | NETIF_F_GSO_UDP_L4); |
3001 | netdev->features |= netdev->hw_features; |
3002 | |
3003 | err = otx2_mcam_flow_init(pf); |
3004 | if (err) |
3005 | goto err_ptp_destroy; |
3006 | |
3007 | err = cn10k_mcs_init(pfvf: pf); |
3008 | if (err) |
3009 | goto err_del_mcam_entries; |
3010 | |
3011 | if (pf->flags & OTX2_FLAG_NTUPLE_SUPPORT) |
3012 | netdev->hw_features |= NETIF_F_NTUPLE; |
3013 | |
3014 | if (pf->flags & OTX2_FLAG_UCAST_FLTR_SUPPORT) |
3015 | netdev->priv_flags |= IFF_UNICAST_FLT; |
3016 | |
3017 | /* Support TSO on tag interface */ |
3018 | netdev->vlan_features |= netdev->features; |
3019 | netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX | |
3020 | NETIF_F_HW_VLAN_STAG_TX; |
3021 | if (pf->flags & OTX2_FLAG_RX_VLAN_SUPPORT) |
3022 | netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX | |
3023 | NETIF_F_HW_VLAN_STAG_RX; |
3024 | netdev->features |= netdev->hw_features; |
3025 | |
3026 | /* HW supports tc offload but mutually exclusive with n-tuple filters */ |
3027 | if (pf->flags & OTX2_FLAG_TC_FLOWER_SUPPORT) |
3028 | netdev->hw_features |= NETIF_F_HW_TC; |
3029 | |
3030 | netdev->hw_features |= NETIF_F_LOOPBACK | NETIF_F_RXALL; |
3031 | |
3032 | netif_set_tso_max_segs(dev: netdev, OTX2_MAX_GSO_SEGS); |
3033 | netdev->watchdog_timeo = OTX2_TX_TIMEOUT; |
3034 | |
3035 | netdev->netdev_ops = &otx2_netdev_ops; |
3036 | netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT; |
3037 | |
3038 | netdev->min_mtu = OTX2_MIN_MTU; |
3039 | netdev->max_mtu = otx2_get_max_mtu(pfvf: pf); |
3040 | |
3041 | err = register_netdev(dev: netdev); |
3042 | if (err) { |
3043 | dev_err(dev, "Failed to register netdevice\n" ); |
3044 | goto err_mcs_free; |
3045 | } |
3046 | |
3047 | err = otx2_wq_init(pf); |
3048 | if (err) |
3049 | goto err_unreg_netdev; |
3050 | |
3051 | otx2_set_ethtool_ops(netdev); |
3052 | |
3053 | err = otx2_init_tc(nic: pf); |
3054 | if (err) |
3055 | goto err_mcam_flow_del; |
3056 | |
3057 | err = otx2_register_dl(pfvf: pf); |
3058 | if (err) |
3059 | goto err_mcam_flow_del; |
3060 | |
3061 | /* Initialize SR-IOV resources */ |
3062 | err = otx2_sriov_vfcfg_init(pf); |
3063 | if (err) |
3064 | goto err_pf_sriov_init; |
3065 | |
3066 | /* Enable link notifications */ |
3067 | otx2_cgx_config_linkevents(pf, enable: true); |
3068 | |
3069 | #ifdef CONFIG_DCB |
3070 | err = otx2_dcbnl_set_ops(dev: netdev); |
3071 | if (err) |
3072 | goto err_pf_sriov_init; |
3073 | #endif |
3074 | |
3075 | otx2_qos_init(pfvf: pf, qos_txqs); |
3076 | |
3077 | return 0; |
3078 | |
3079 | err_pf_sriov_init: |
3080 | otx2_shutdown_tc(nic: pf); |
3081 | err_mcam_flow_del: |
3082 | otx2_mcam_flow_del(pf); |
3083 | err_unreg_netdev: |
3084 | unregister_netdev(dev: netdev); |
3085 | err_mcs_free: |
3086 | cn10k_mcs_free(pfvf: pf); |
3087 | err_del_mcam_entries: |
3088 | otx2_mcam_flow_del(pf); |
3089 | err_ptp_destroy: |
3090 | otx2_ptp_destroy(pfvf: pf); |
3091 | err_detach_rsrc: |
3092 | if (pf->hw.lmt_info) |
3093 | free_percpu(pdata: pf->hw.lmt_info); |
3094 | if (test_bit(CN10K_LMTST, &pf->hw.cap_flag)) |
3095 | qmem_free(dev: pf->dev, qmem: pf->dync_lmt); |
3096 | otx2_detach_resources(mbox: &pf->mbox); |
3097 | err_disable_mbox_intr: |
3098 | otx2_disable_mbox_intr(pf); |
3099 | err_mbox_destroy: |
3100 | otx2_pfaf_mbox_destroy(pf); |
3101 | err_free_irq_vectors: |
3102 | pci_free_irq_vectors(dev: hw->pdev); |
3103 | err_free_netdev: |
3104 | pci_set_drvdata(pdev, NULL); |
3105 | free_netdev(dev: netdev); |
3106 | err_release_regions: |
3107 | pci_release_regions(pdev); |
3108 | return err; |
3109 | } |
3110 | |
3111 | static void otx2_vf_link_event_task(struct work_struct *work) |
3112 | { |
3113 | struct otx2_vf_config *config; |
3114 | struct cgx_link_info_msg *req; |
3115 | struct mbox_msghdr *msghdr; |
3116 | struct delayed_work *dwork; |
3117 | struct otx2_nic *pf; |
3118 | int vf_idx; |
3119 | |
3120 | config = container_of(work, struct otx2_vf_config, |
3121 | link_event_work.work); |
3122 | vf_idx = config - config->pf->vf_configs; |
3123 | pf = config->pf; |
3124 | |
3125 | if (config->intf_down) |
3126 | return; |
3127 | |
3128 | mutex_lock(&pf->mbox.lock); |
3129 | |
3130 | dwork = &config->link_event_work; |
3131 | |
3132 | if (!otx2_mbox_wait_for_zero(mbox: &pf->mbox_pfvf[0].mbox_up, devid: vf_idx)) { |
3133 | schedule_delayed_work(dwork, delay: msecs_to_jiffies(m: 100)); |
3134 | mutex_unlock(lock: &pf->mbox.lock); |
3135 | return; |
3136 | } |
3137 | |
3138 | msghdr = otx2_mbox_alloc_msg_rsp(mbox: &pf->mbox_pfvf[0].mbox_up, devid: vf_idx, |
3139 | size: sizeof(*req), size_rsp: sizeof(struct msg_rsp)); |
3140 | if (!msghdr) { |
3141 | dev_err(pf->dev, "Failed to create VF%d link event\n" , vf_idx); |
3142 | mutex_unlock(lock: &pf->mbox.lock); |
3143 | return; |
3144 | } |
3145 | |
3146 | req = (struct cgx_link_info_msg *)msghdr; |
3147 | req->hdr.id = MBOX_MSG_CGX_LINK_EVENT; |
3148 | req->hdr.sig = OTX2_MBOX_REQ_SIG; |
3149 | memcpy(&req->link_info, &pf->linfo, sizeof(req->link_info)); |
3150 | |
3151 | otx2_mbox_wait_for_zero(mbox: &pf->mbox_pfvf[0].mbox_up, devid: vf_idx); |
3152 | |
3153 | otx2_sync_mbox_up_msg(mbox: &pf->mbox_pfvf[0], devid: vf_idx); |
3154 | |
3155 | mutex_unlock(lock: &pf->mbox.lock); |
3156 | } |
3157 | |
3158 | static int otx2_sriov_enable(struct pci_dev *pdev, int numvfs) |
3159 | { |
3160 | struct net_device *netdev = pci_get_drvdata(pdev); |
3161 | struct otx2_nic *pf = netdev_priv(dev: netdev); |
3162 | int ret; |
3163 | |
3164 | /* Init PF <=> VF mailbox stuff */ |
3165 | ret = otx2_pfvf_mbox_init(pf, numvfs); |
3166 | if (ret) |
3167 | return ret; |
3168 | |
3169 | ret = otx2_register_pfvf_mbox_intr(pf, numvfs); |
3170 | if (ret) |
3171 | goto free_mbox; |
3172 | |
3173 | ret = otx2_pf_flr_init(pf, num_vfs: numvfs); |
3174 | if (ret) |
3175 | goto free_intr; |
3176 | |
3177 | ret = otx2_register_flr_me_intr(pf, numvfs); |
3178 | if (ret) |
3179 | goto free_flr; |
3180 | |
3181 | ret = pci_enable_sriov(dev: pdev, nr_virtfn: numvfs); |
3182 | if (ret) |
3183 | goto free_flr_intr; |
3184 | |
3185 | return numvfs; |
3186 | free_flr_intr: |
3187 | otx2_disable_flr_me_intr(pf); |
3188 | free_flr: |
3189 | otx2_flr_wq_destroy(pf); |
3190 | free_intr: |
3191 | otx2_disable_pfvf_mbox_intr(pf, numvfs); |
3192 | free_mbox: |
3193 | otx2_pfvf_mbox_destroy(pf); |
3194 | return ret; |
3195 | } |
3196 | |
3197 | static int otx2_sriov_disable(struct pci_dev *pdev) |
3198 | { |
3199 | struct net_device *netdev = pci_get_drvdata(pdev); |
3200 | struct otx2_nic *pf = netdev_priv(dev: netdev); |
3201 | int numvfs = pci_num_vf(dev: pdev); |
3202 | |
3203 | if (!numvfs) |
3204 | return 0; |
3205 | |
3206 | pci_disable_sriov(dev: pdev); |
3207 | |
3208 | otx2_disable_flr_me_intr(pf); |
3209 | otx2_flr_wq_destroy(pf); |
3210 | otx2_disable_pfvf_mbox_intr(pf, numvfs); |
3211 | otx2_pfvf_mbox_destroy(pf); |
3212 | |
3213 | return 0; |
3214 | } |
3215 | |
3216 | static int otx2_sriov_configure(struct pci_dev *pdev, int numvfs) |
3217 | { |
3218 | if (numvfs == 0) |
3219 | return otx2_sriov_disable(pdev); |
3220 | else |
3221 | return otx2_sriov_enable(pdev, numvfs); |
3222 | } |
3223 | |
3224 | static void otx2_remove(struct pci_dev *pdev) |
3225 | { |
3226 | struct net_device *netdev = pci_get_drvdata(pdev); |
3227 | struct otx2_nic *pf; |
3228 | |
3229 | if (!netdev) |
3230 | return; |
3231 | |
3232 | pf = netdev_priv(dev: netdev); |
3233 | |
3234 | pf->flags |= OTX2_FLAG_PF_SHUTDOWN; |
3235 | |
3236 | if (pf->flags & OTX2_FLAG_TX_TSTAMP_ENABLED) |
3237 | otx2_config_hw_tx_tstamp(pfvf: pf, enable: false); |
3238 | if (pf->flags & OTX2_FLAG_RX_TSTAMP_ENABLED) |
3239 | otx2_config_hw_rx_tstamp(pfvf: pf, enable: false); |
3240 | |
3241 | /* Disable 802.3x pause frames */ |
3242 | if (pf->flags & OTX2_FLAG_RX_PAUSE_ENABLED || |
3243 | (pf->flags & OTX2_FLAG_TX_PAUSE_ENABLED)) { |
3244 | pf->flags &= ~OTX2_FLAG_RX_PAUSE_ENABLED; |
3245 | pf->flags &= ~OTX2_FLAG_TX_PAUSE_ENABLED; |
3246 | otx2_config_pause_frm(pfvf: pf); |
3247 | } |
3248 | |
3249 | #ifdef CONFIG_DCB |
3250 | /* Disable PFC config */ |
3251 | if (pf->pfc_en) { |
3252 | pf->pfc_en = 0; |
3253 | otx2_config_priority_flow_ctrl(pfvf: pf); |
3254 | } |
3255 | #endif |
3256 | cancel_work_sync(work: &pf->reset_task); |
3257 | /* Disable link notifications */ |
3258 | otx2_cgx_config_linkevents(pf, enable: false); |
3259 | |
3260 | otx2_unregister_dl(pfvf: pf); |
3261 | unregister_netdev(dev: netdev); |
3262 | cn10k_mcs_free(pfvf: pf); |
3263 | otx2_sriov_disable(pdev: pf->pdev); |
3264 | otx2_sriov_vfcfg_cleanup(pf); |
3265 | if (pf->otx2_wq) |
3266 | destroy_workqueue(wq: pf->otx2_wq); |
3267 | |
3268 | otx2_ptp_destroy(pfvf: pf); |
3269 | otx2_mcam_flow_del(pf); |
3270 | otx2_shutdown_tc(nic: pf); |
3271 | otx2_shutdown_qos(pfvf: pf); |
3272 | otx2_detach_resources(mbox: &pf->mbox); |
3273 | if (pf->hw.lmt_info) |
3274 | free_percpu(pdata: pf->hw.lmt_info); |
3275 | if (test_bit(CN10K_LMTST, &pf->hw.cap_flag)) |
3276 | qmem_free(dev: pf->dev, qmem: pf->dync_lmt); |
3277 | otx2_disable_mbox_intr(pf); |
3278 | otx2_pfaf_mbox_destroy(pf); |
3279 | pci_free_irq_vectors(dev: pf->pdev); |
3280 | pci_set_drvdata(pdev, NULL); |
3281 | free_netdev(dev: netdev); |
3282 | |
3283 | pci_release_regions(pdev); |
3284 | } |
3285 | |
3286 | static struct pci_driver otx2_pf_driver = { |
3287 | .name = DRV_NAME, |
3288 | .id_table = otx2_pf_id_table, |
3289 | .probe = otx2_probe, |
3290 | .shutdown = otx2_remove, |
3291 | .remove = otx2_remove, |
3292 | .sriov_configure = otx2_sriov_configure |
3293 | }; |
3294 | |
3295 | static int __init otx2_rvupf_init_module(void) |
3296 | { |
3297 | pr_info("%s: %s\n" , DRV_NAME, DRV_STRING); |
3298 | |
3299 | return pci_register_driver(&otx2_pf_driver); |
3300 | } |
3301 | |
3302 | static void __exit otx2_rvupf_cleanup_module(void) |
3303 | { |
3304 | pci_unregister_driver(dev: &otx2_pf_driver); |
3305 | } |
3306 | |
3307 | module_init(otx2_rvupf_init_module); |
3308 | module_exit(otx2_rvupf_cleanup_module); |
3309 | |