1 | /* |
2 | * Linux driver for VMware's vmxnet3 ethernet NIC. |
3 | * |
4 | * Copyright (C) 2008-2022, VMware, Inc. All Rights Reserved. |
5 | * |
6 | * This program is free software; you can redistribute it and/or modify it |
7 | * under the terms of the GNU General Public License as published by the |
8 | * Free Software Foundation; version 2 of the License and no later version. |
9 | * |
10 | * This program is distributed in the hope that it will be useful, but |
11 | * WITHOUT ANY WARRANTY; without even the implied warranty of |
12 | * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or |
13 | * NON INFRINGEMENT. See the GNU General Public License for more |
14 | * details. |
15 | * |
16 | * You should have received a copy of the GNU General Public License |
17 | * along with this program; if not, write to the Free Software |
18 | * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
19 | * |
20 | * The full GNU General Public License is included in this distribution in |
21 | * the file called "COPYING". |
22 | * |
23 | * Maintained by: pv-drivers@vmware.com |
24 | * |
25 | */ |
26 | |
27 | #include <linux/module.h> |
28 | #include <net/ip6_checksum.h> |
29 | |
30 | #include "vmxnet3_int.h" |
31 | #include "vmxnet3_xdp.h" |
32 | |
33 | char vmxnet3_driver_name[] = "vmxnet3" ; |
34 | #define VMXNET3_DRIVER_DESC "VMware vmxnet3 virtual NIC driver" |
35 | |
36 | /* |
37 | * PCI Device ID Table |
38 | * Last entry must be all 0s |
39 | */ |
40 | static const struct pci_device_id vmxnet3_pciid_table[] = { |
41 | {PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_VMXNET3)}, |
42 | {0} |
43 | }; |
44 | |
45 | MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table); |
46 | |
47 | static int enable_mq = 1; |
48 | |
49 | static void |
50 | vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, const u8 *mac); |
51 | |
52 | /* |
53 | * Enable/Disable the given intr |
54 | */ |
55 | static void |
56 | vmxnet3_enable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx) |
57 | { |
58 | VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 0); |
59 | } |
60 | |
61 | |
62 | static void |
63 | vmxnet3_disable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx) |
64 | { |
65 | VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 1); |
66 | } |
67 | |
68 | |
69 | /* |
70 | * Enable/Disable all intrs used by the device |
71 | */ |
72 | static void |
73 | vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter) |
74 | { |
75 | int i; |
76 | |
77 | for (i = 0; i < adapter->intr.num_intrs; i++) |
78 | vmxnet3_enable_intr(adapter, intr_idx: i); |
79 | if (!VMXNET3_VERSION_GE_6(adapter) || |
80 | !adapter->queuesExtEnabled) { |
81 | adapter->shared->devRead.intrConf.intrCtrl &= |
82 | cpu_to_le32(~VMXNET3_IC_DISABLE_ALL); |
83 | } else { |
84 | adapter->shared->devReadExt.intrConfExt.intrCtrl &= |
85 | cpu_to_le32(~VMXNET3_IC_DISABLE_ALL); |
86 | } |
87 | } |
88 | |
89 | |
90 | static void |
91 | vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter) |
92 | { |
93 | int i; |
94 | |
95 | if (!VMXNET3_VERSION_GE_6(adapter) || |
96 | !adapter->queuesExtEnabled) { |
97 | adapter->shared->devRead.intrConf.intrCtrl |= |
98 | cpu_to_le32(VMXNET3_IC_DISABLE_ALL); |
99 | } else { |
100 | adapter->shared->devReadExt.intrConfExt.intrCtrl |= |
101 | cpu_to_le32(VMXNET3_IC_DISABLE_ALL); |
102 | } |
103 | for (i = 0; i < adapter->intr.num_intrs; i++) |
104 | vmxnet3_disable_intr(adapter, intr_idx: i); |
105 | } |
106 | |
107 | |
108 | static void |
109 | vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events) |
110 | { |
111 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_ECR, events); |
112 | } |
113 | |
114 | |
115 | static bool |
116 | vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) |
117 | { |
118 | return tq->stopped; |
119 | } |
120 | |
121 | |
122 | static void |
123 | vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) |
124 | { |
125 | tq->stopped = false; |
126 | netif_start_subqueue(dev: adapter->netdev, queue_index: tq - adapter->tx_queue); |
127 | } |
128 | |
129 | |
130 | static void |
131 | vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) |
132 | { |
133 | tq->stopped = false; |
134 | netif_wake_subqueue(dev: adapter->netdev, queue_index: (tq - adapter->tx_queue)); |
135 | } |
136 | |
137 | |
138 | static void |
139 | vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) |
140 | { |
141 | tq->stopped = true; |
142 | tq->num_stop++; |
143 | netif_stop_subqueue(dev: adapter->netdev, queue_index: (tq - adapter->tx_queue)); |
144 | } |
145 | |
146 | /* Check if capability is supported by UPT device or |
147 | * UPT is even requested |
148 | */ |
149 | bool |
150 | vmxnet3_check_ptcapability(u32 cap_supported, u32 cap) |
151 | { |
152 | if (cap_supported & (1UL << VMXNET3_DCR_ERROR) || |
153 | cap_supported & (1UL << cap)) { |
154 | return true; |
155 | } |
156 | |
157 | return false; |
158 | } |
159 | |
160 | |
161 | /* |
162 | * Check the link state. This may start or stop the tx queue. |
163 | */ |
164 | static void |
165 | vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue) |
166 | { |
167 | u32 ret; |
168 | int i; |
169 | unsigned long flags; |
170 | |
171 | spin_lock_irqsave(&adapter->cmd_lock, flags); |
172 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK); |
173 | ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); |
174 | spin_unlock_irqrestore(lock: &adapter->cmd_lock, flags); |
175 | |
176 | adapter->link_speed = ret >> 16; |
177 | if (ret & 1) { /* Link is up. */ |
178 | netdev_info(dev: adapter->netdev, format: "NIC Link is Up %d Mbps\n" , |
179 | adapter->link_speed); |
180 | netif_carrier_on(dev: adapter->netdev); |
181 | |
182 | if (affectTxQueue) { |
183 | for (i = 0; i < adapter->num_tx_queues; i++) |
184 | vmxnet3_tq_start(tq: &adapter->tx_queue[i], |
185 | adapter); |
186 | } |
187 | } else { |
188 | netdev_info(dev: adapter->netdev, format: "NIC Link is Down\n" ); |
189 | netif_carrier_off(dev: adapter->netdev); |
190 | |
191 | if (affectTxQueue) { |
192 | for (i = 0; i < adapter->num_tx_queues; i++) |
193 | vmxnet3_tq_stop(tq: &adapter->tx_queue[i], adapter); |
194 | } |
195 | } |
196 | } |
197 | |
198 | static void |
199 | vmxnet3_process_events(struct vmxnet3_adapter *adapter) |
200 | { |
201 | int i; |
202 | unsigned long flags; |
203 | u32 events = le32_to_cpu(adapter->shared->ecr); |
204 | if (!events) |
205 | return; |
206 | |
207 | vmxnet3_ack_events(adapter, events); |
208 | |
209 | /* Check if link state has changed */ |
210 | if (events & VMXNET3_ECR_LINK) |
211 | vmxnet3_check_link(adapter, affectTxQueue: true); |
212 | |
213 | /* Check if there is an error on xmit/recv queues */ |
214 | if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) { |
215 | spin_lock_irqsave(&adapter->cmd_lock, flags); |
216 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, |
217 | VMXNET3_CMD_GET_QUEUE_STATUS); |
218 | spin_unlock_irqrestore(lock: &adapter->cmd_lock, flags); |
219 | |
220 | for (i = 0; i < adapter->num_tx_queues; i++) |
221 | if (adapter->tqd_start[i].status.stopped) |
222 | dev_err(&adapter->netdev->dev, |
223 | "%s: tq[%d] error 0x%x\n" , |
224 | adapter->netdev->name, i, le32_to_cpu( |
225 | adapter->tqd_start[i].status.error)); |
226 | for (i = 0; i < adapter->num_rx_queues; i++) |
227 | if (adapter->rqd_start[i].status.stopped) |
228 | dev_err(&adapter->netdev->dev, |
229 | "%s: rq[%d] error 0x%x\n" , |
230 | adapter->netdev->name, i, |
231 | adapter->rqd_start[i].status.error); |
232 | |
233 | schedule_work(work: &adapter->work); |
234 | } |
235 | } |
236 | |
237 | #ifdef __BIG_ENDIAN_BITFIELD |
238 | /* |
239 | * The device expects the bitfields in shared structures to be written in |
240 | * little endian. When CPU is big endian, the following routines are used to |
241 | * correctly read and write into ABI. |
242 | * The general technique used here is : double word bitfields are defined in |
243 | * opposite order for big endian architecture. Then before reading them in |
244 | * driver the complete double word is translated using le32_to_cpu. Similarly |
245 | * After the driver writes into bitfields, cpu_to_le32 is used to translate the |
246 | * double words into required format. |
247 | * In order to avoid touching bits in shared structure more than once, temporary |
248 | * descriptors are used. These are passed as srcDesc to following functions. |
249 | */ |
250 | static void vmxnet3_RxDescToCPU(const struct Vmxnet3_RxDesc *srcDesc, |
251 | struct Vmxnet3_RxDesc *dstDesc) |
252 | { |
253 | u32 *src = (u32 *)srcDesc + 2; |
254 | u32 *dst = (u32 *)dstDesc + 2; |
255 | dstDesc->addr = le64_to_cpu(srcDesc->addr); |
256 | *dst = le32_to_cpu(*src); |
257 | dstDesc->ext1 = le32_to_cpu(srcDesc->ext1); |
258 | } |
259 | |
260 | static void vmxnet3_TxDescToLe(const struct Vmxnet3_TxDesc *srcDesc, |
261 | struct Vmxnet3_TxDesc *dstDesc) |
262 | { |
263 | int i; |
264 | u32 *src = (u32 *)(srcDesc + 1); |
265 | u32 *dst = (u32 *)(dstDesc + 1); |
266 | |
267 | /* Working backwards so that the gen bit is set at the end. */ |
268 | for (i = 2; i > 0; i--) { |
269 | src--; |
270 | dst--; |
271 | *dst = cpu_to_le32(*src); |
272 | } |
273 | } |
274 | |
275 | |
276 | static void vmxnet3_RxCompToCPU(const struct Vmxnet3_RxCompDesc *srcDesc, |
277 | struct Vmxnet3_RxCompDesc *dstDesc) |
278 | { |
279 | int i = 0; |
280 | u32 *src = (u32 *)srcDesc; |
281 | u32 *dst = (u32 *)dstDesc; |
282 | for (i = 0; i < sizeof(struct Vmxnet3_RxCompDesc) / sizeof(u32); i++) { |
283 | *dst = le32_to_cpu(*src); |
284 | src++; |
285 | dst++; |
286 | } |
287 | } |
288 | |
289 | |
290 | /* Used to read bitfield values from double words. */ |
291 | static u32 get_bitfield32(const __le32 *bitfield, u32 pos, u32 size) |
292 | { |
293 | u32 temp = le32_to_cpu(*bitfield); |
294 | u32 mask = ((1 << size) - 1) << pos; |
295 | temp &= mask; |
296 | temp >>= pos; |
297 | return temp; |
298 | } |
299 | |
300 | |
301 | |
302 | #endif /* __BIG_ENDIAN_BITFIELD */ |
303 | |
304 | #ifdef __BIG_ENDIAN_BITFIELD |
305 | |
306 | # define VMXNET3_TXDESC_GET_GEN(txdesc) get_bitfield32(((const __le32 *) \ |
307 | txdesc) + VMXNET3_TXD_GEN_DWORD_SHIFT, \ |
308 | VMXNET3_TXD_GEN_SHIFT, VMXNET3_TXD_GEN_SIZE) |
309 | # define VMXNET3_TXDESC_GET_EOP(txdesc) get_bitfield32(((const __le32 *) \ |
310 | txdesc) + VMXNET3_TXD_EOP_DWORD_SHIFT, \ |
311 | VMXNET3_TXD_EOP_SHIFT, VMXNET3_TXD_EOP_SIZE) |
312 | # define VMXNET3_TCD_GET_GEN(tcd) get_bitfield32(((const __le32 *)tcd) + \ |
313 | VMXNET3_TCD_GEN_DWORD_SHIFT, VMXNET3_TCD_GEN_SHIFT, \ |
314 | VMXNET3_TCD_GEN_SIZE) |
315 | # define VMXNET3_TCD_GET_TXIDX(tcd) get_bitfield32((const __le32 *)tcd, \ |
316 | VMXNET3_TCD_TXIDX_SHIFT, VMXNET3_TCD_TXIDX_SIZE) |
317 | # define vmxnet3_getRxComp(dstrcd, rcd, tmp) do { \ |
318 | (dstrcd) = (tmp); \ |
319 | vmxnet3_RxCompToCPU((rcd), (tmp)); \ |
320 | } while (0) |
321 | # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) do { \ |
322 | (dstrxd) = (tmp); \ |
323 | vmxnet3_RxDescToCPU((rxd), (tmp)); \ |
324 | } while (0) |
325 | |
326 | #else |
327 | |
328 | # define VMXNET3_TXDESC_GET_GEN(txdesc) ((txdesc)->gen) |
329 | # define VMXNET3_TXDESC_GET_EOP(txdesc) ((txdesc)->eop) |
330 | # define VMXNET3_TCD_GET_GEN(tcd) ((tcd)->gen) |
331 | # define VMXNET3_TCD_GET_TXIDX(tcd) ((tcd)->txdIdx) |
332 | # define vmxnet3_getRxComp(dstrcd, rcd, tmp) (dstrcd) = (rcd) |
333 | # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) (dstrxd) = (rxd) |
334 | |
335 | #endif /* __BIG_ENDIAN_BITFIELD */ |
336 | |
337 | |
338 | static void |
339 | vmxnet3_unmap_tx_buf(struct vmxnet3_tx_buf_info *tbi, |
340 | struct pci_dev *pdev) |
341 | { |
342 | u32 map_type = tbi->map_type; |
343 | |
344 | if (map_type & VMXNET3_MAP_SINGLE) |
345 | dma_unmap_single(&pdev->dev, tbi->dma_addr, tbi->len, |
346 | DMA_TO_DEVICE); |
347 | else if (map_type & VMXNET3_MAP_PAGE) |
348 | dma_unmap_page(&pdev->dev, tbi->dma_addr, tbi->len, |
349 | DMA_TO_DEVICE); |
350 | else |
351 | BUG_ON(map_type & ~VMXNET3_MAP_XDP); |
352 | |
353 | tbi->map_type = VMXNET3_MAP_NONE; /* to help debugging */ |
354 | } |
355 | |
356 | |
357 | static int |
358 | vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq, |
359 | struct pci_dev *pdev, struct vmxnet3_adapter *adapter, |
360 | struct xdp_frame_bulk *bq) |
361 | { |
362 | struct vmxnet3_tx_buf_info *tbi; |
363 | int entries = 0; |
364 | u32 map_type; |
365 | |
366 | /* no out of order completion */ |
367 | BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp); |
368 | BUG_ON(VMXNET3_TXDESC_GET_EOP(&(tq->tx_ring.base[eop_idx].txd)) != 1); |
369 | |
370 | tbi = &tq->buf_info[eop_idx]; |
371 | BUG_ON(!tbi->skb); |
372 | map_type = tbi->map_type; |
373 | VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size); |
374 | |
375 | while (tq->tx_ring.next2comp != eop_idx) { |
376 | vmxnet3_unmap_tx_buf(tbi: tq->buf_info + tq->tx_ring.next2comp, |
377 | pdev); |
378 | |
379 | /* update next2comp w/o tx_lock. Since we are marking more, |
380 | * instead of less, tx ring entries avail, the worst case is |
381 | * that the tx routine incorrectly re-queues a pkt due to |
382 | * insufficient tx ring entries. |
383 | */ |
384 | vmxnet3_cmd_ring_adv_next2comp(ring: &tq->tx_ring); |
385 | entries++; |
386 | } |
387 | |
388 | if (map_type & VMXNET3_MAP_XDP) |
389 | xdp_return_frame_bulk(xdpf: tbi->xdpf, bq); |
390 | else |
391 | dev_kfree_skb_any(skb: tbi->skb); |
392 | |
393 | /* xdpf and skb are in an anonymous union. */ |
394 | tbi->skb = NULL; |
395 | |
396 | return entries; |
397 | } |
398 | |
399 | |
400 | static int |
401 | vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq, |
402 | struct vmxnet3_adapter *adapter) |
403 | { |
404 | union Vmxnet3_GenericDesc *gdesc; |
405 | struct xdp_frame_bulk bq; |
406 | int completed = 0; |
407 | |
408 | xdp_frame_bulk_init(bq: &bq); |
409 | rcu_read_lock(); |
410 | |
411 | gdesc = tq->comp_ring.base + tq->comp_ring.next2proc; |
412 | while (VMXNET3_TCD_GET_GEN(&gdesc->tcd) == tq->comp_ring.gen) { |
413 | /* Prevent any &gdesc->tcd field from being (speculatively) |
414 | * read before (&gdesc->tcd)->gen is read. |
415 | */ |
416 | dma_rmb(); |
417 | |
418 | completed += vmxnet3_unmap_pkt(VMXNET3_TCD_GET_TXIDX( |
419 | &gdesc->tcd), tq, pdev: adapter->pdev, |
420 | adapter, bq: &bq); |
421 | |
422 | vmxnet3_comp_ring_adv_next2proc(ring: &tq->comp_ring); |
423 | gdesc = tq->comp_ring.base + tq->comp_ring.next2proc; |
424 | } |
425 | xdp_flush_frame_bulk(bq: &bq); |
426 | rcu_read_unlock(); |
427 | |
428 | if (completed) { |
429 | spin_lock(lock: &tq->tx_lock); |
430 | if (unlikely(vmxnet3_tq_stopped(tq, adapter) && |
431 | vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) > |
432 | VMXNET3_WAKE_QUEUE_THRESHOLD(tq) && |
433 | netif_carrier_ok(adapter->netdev))) { |
434 | vmxnet3_tq_wake(tq, adapter); |
435 | } |
436 | spin_unlock(lock: &tq->tx_lock); |
437 | } |
438 | return completed; |
439 | } |
440 | |
441 | |
442 | static void |
443 | vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq, |
444 | struct vmxnet3_adapter *adapter) |
445 | { |
446 | struct xdp_frame_bulk bq; |
447 | u32 map_type; |
448 | int i; |
449 | |
450 | xdp_frame_bulk_init(bq: &bq); |
451 | rcu_read_lock(); |
452 | |
453 | while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) { |
454 | struct vmxnet3_tx_buf_info *tbi; |
455 | |
456 | tbi = tq->buf_info + tq->tx_ring.next2comp; |
457 | map_type = tbi->map_type; |
458 | |
459 | vmxnet3_unmap_tx_buf(tbi, pdev: adapter->pdev); |
460 | if (tbi->skb) { |
461 | if (map_type & VMXNET3_MAP_XDP) |
462 | xdp_return_frame_bulk(xdpf: tbi->xdpf, bq: &bq); |
463 | else |
464 | dev_kfree_skb_any(skb: tbi->skb); |
465 | tbi->skb = NULL; |
466 | } |
467 | vmxnet3_cmd_ring_adv_next2comp(ring: &tq->tx_ring); |
468 | } |
469 | |
470 | xdp_flush_frame_bulk(bq: &bq); |
471 | rcu_read_unlock(); |
472 | |
473 | /* sanity check, verify all buffers are indeed unmapped */ |
474 | for (i = 0; i < tq->tx_ring.size; i++) |
475 | BUG_ON(tq->buf_info[i].map_type != VMXNET3_MAP_NONE); |
476 | |
477 | tq->tx_ring.gen = VMXNET3_INIT_GEN; |
478 | tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0; |
479 | |
480 | tq->comp_ring.gen = VMXNET3_INIT_GEN; |
481 | tq->comp_ring.next2proc = 0; |
482 | } |
483 | |
484 | |
485 | static void |
486 | vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq, |
487 | struct vmxnet3_adapter *adapter) |
488 | { |
489 | if (tq->tx_ring.base) { |
490 | dma_free_coherent(dev: &adapter->pdev->dev, size: tq->tx_ring.size * |
491 | sizeof(struct Vmxnet3_TxDesc), |
492 | cpu_addr: tq->tx_ring.base, dma_handle: tq->tx_ring.basePA); |
493 | tq->tx_ring.base = NULL; |
494 | } |
495 | if (tq->data_ring.base) { |
496 | dma_free_coherent(dev: &adapter->pdev->dev, |
497 | size: tq->data_ring.size * tq->txdata_desc_size, |
498 | cpu_addr: tq->data_ring.base, dma_handle: tq->data_ring.basePA); |
499 | tq->data_ring.base = NULL; |
500 | } |
501 | if (tq->comp_ring.base) { |
502 | dma_free_coherent(dev: &adapter->pdev->dev, size: tq->comp_ring.size * |
503 | sizeof(struct Vmxnet3_TxCompDesc), |
504 | cpu_addr: tq->comp_ring.base, dma_handle: tq->comp_ring.basePA); |
505 | tq->comp_ring.base = NULL; |
506 | } |
507 | kfree(objp: tq->buf_info); |
508 | tq->buf_info = NULL; |
509 | } |
510 | |
511 | |
512 | /* Destroy all tx queues */ |
513 | void |
514 | vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter) |
515 | { |
516 | int i; |
517 | |
518 | for (i = 0; i < adapter->num_tx_queues; i++) |
519 | vmxnet3_tq_destroy(tq: &adapter->tx_queue[i], adapter); |
520 | } |
521 | |
522 | |
523 | static void |
524 | vmxnet3_tq_init(struct vmxnet3_tx_queue *tq, |
525 | struct vmxnet3_adapter *adapter) |
526 | { |
527 | int i; |
528 | |
529 | /* reset the tx ring contents to 0 and reset the tx ring states */ |
530 | memset(tq->tx_ring.base, 0, tq->tx_ring.size * |
531 | sizeof(struct Vmxnet3_TxDesc)); |
532 | tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0; |
533 | tq->tx_ring.gen = VMXNET3_INIT_GEN; |
534 | |
535 | memset(tq->data_ring.base, 0, |
536 | tq->data_ring.size * tq->txdata_desc_size); |
537 | |
538 | /* reset the tx comp ring contents to 0 and reset comp ring states */ |
539 | memset(tq->comp_ring.base, 0, tq->comp_ring.size * |
540 | sizeof(struct Vmxnet3_TxCompDesc)); |
541 | tq->comp_ring.next2proc = 0; |
542 | tq->comp_ring.gen = VMXNET3_INIT_GEN; |
543 | |
544 | /* reset the bookkeeping data */ |
545 | memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size); |
546 | for (i = 0; i < tq->tx_ring.size; i++) |
547 | tq->buf_info[i].map_type = VMXNET3_MAP_NONE; |
548 | |
549 | /* stats are not reset */ |
550 | } |
551 | |
552 | |
553 | static int |
554 | vmxnet3_tq_create(struct vmxnet3_tx_queue *tq, |
555 | struct vmxnet3_adapter *adapter) |
556 | { |
557 | BUG_ON(tq->tx_ring.base || tq->data_ring.base || |
558 | tq->comp_ring.base || tq->buf_info); |
559 | |
560 | tq->tx_ring.base = dma_alloc_coherent(dev: &adapter->pdev->dev, |
561 | size: tq->tx_ring.size * sizeof(struct Vmxnet3_TxDesc), |
562 | dma_handle: &tq->tx_ring.basePA, GFP_KERNEL); |
563 | if (!tq->tx_ring.base) { |
564 | netdev_err(dev: adapter->netdev, format: "failed to allocate tx ring\n" ); |
565 | goto err; |
566 | } |
567 | |
568 | tq->data_ring.base = dma_alloc_coherent(dev: &adapter->pdev->dev, |
569 | size: tq->data_ring.size * tq->txdata_desc_size, |
570 | dma_handle: &tq->data_ring.basePA, GFP_KERNEL); |
571 | if (!tq->data_ring.base) { |
572 | netdev_err(dev: adapter->netdev, format: "failed to allocate tx data ring\n" ); |
573 | goto err; |
574 | } |
575 | |
576 | tq->comp_ring.base = dma_alloc_coherent(dev: &adapter->pdev->dev, |
577 | size: tq->comp_ring.size * sizeof(struct Vmxnet3_TxCompDesc), |
578 | dma_handle: &tq->comp_ring.basePA, GFP_KERNEL); |
579 | if (!tq->comp_ring.base) { |
580 | netdev_err(dev: adapter->netdev, format: "failed to allocate tx comp ring\n" ); |
581 | goto err; |
582 | } |
583 | |
584 | tq->buf_info = kcalloc_node(n: tq->tx_ring.size, size: sizeof(tq->buf_info[0]), |
585 | GFP_KERNEL, |
586 | node: dev_to_node(dev: &adapter->pdev->dev)); |
587 | if (!tq->buf_info) |
588 | goto err; |
589 | |
590 | return 0; |
591 | |
592 | err: |
593 | vmxnet3_tq_destroy(tq, adapter); |
594 | return -ENOMEM; |
595 | } |
596 | |
597 | static void |
598 | vmxnet3_tq_cleanup_all(struct vmxnet3_adapter *adapter) |
599 | { |
600 | int i; |
601 | |
602 | for (i = 0; i < adapter->num_tx_queues; i++) |
603 | vmxnet3_tq_cleanup(tq: &adapter->tx_queue[i], adapter); |
604 | } |
605 | |
606 | /* |
607 | * starting from ring->next2fill, allocate rx buffers for the given ring |
608 | * of the rx queue and update the rx desc. stop after @num_to_alloc buffers |
609 | * are allocated or allocation fails |
610 | */ |
611 | |
612 | static int |
613 | vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx, |
614 | int num_to_alloc, struct vmxnet3_adapter *adapter) |
615 | { |
616 | int num_allocated = 0; |
617 | struct vmxnet3_rx_buf_info *rbi_base = rq->buf_info[ring_idx]; |
618 | struct vmxnet3_cmd_ring *ring = &rq->rx_ring[ring_idx]; |
619 | u32 val; |
620 | |
621 | while (num_allocated <= num_to_alloc) { |
622 | struct vmxnet3_rx_buf_info *rbi; |
623 | union Vmxnet3_GenericDesc *gd; |
624 | |
625 | rbi = rbi_base + ring->next2fill; |
626 | gd = ring->base + ring->next2fill; |
627 | rbi->comp_state = VMXNET3_RXD_COMP_PENDING; |
628 | |
629 | if (rbi->buf_type == VMXNET3_RX_BUF_XDP) { |
630 | void *data = vmxnet3_pp_get_buff(pp: rq->page_pool, |
631 | dma_addr: &rbi->dma_addr, |
632 | GFP_KERNEL); |
633 | if (!data) { |
634 | rq->stats.rx_buf_alloc_failure++; |
635 | break; |
636 | } |
637 | rbi->page = virt_to_page(data); |
638 | val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT; |
639 | } else if (rbi->buf_type == VMXNET3_RX_BUF_SKB) { |
640 | if (rbi->skb == NULL) { |
641 | rbi->skb = __netdev_alloc_skb_ip_align(dev: adapter->netdev, |
642 | length: rbi->len, |
643 | GFP_KERNEL); |
644 | if (unlikely(rbi->skb == NULL)) { |
645 | rq->stats.rx_buf_alloc_failure++; |
646 | break; |
647 | } |
648 | |
649 | rbi->dma_addr = dma_map_single( |
650 | &adapter->pdev->dev, |
651 | rbi->skb->data, rbi->len, |
652 | DMA_FROM_DEVICE); |
653 | if (dma_mapping_error(dev: &adapter->pdev->dev, |
654 | dma_addr: rbi->dma_addr)) { |
655 | dev_kfree_skb_any(skb: rbi->skb); |
656 | rbi->skb = NULL; |
657 | rq->stats.rx_buf_alloc_failure++; |
658 | break; |
659 | } |
660 | } else { |
661 | /* rx buffer skipped by the device */ |
662 | } |
663 | val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT; |
664 | } else { |
665 | BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE || |
666 | rbi->len != PAGE_SIZE); |
667 | |
668 | if (rbi->page == NULL) { |
669 | rbi->page = alloc_page(GFP_ATOMIC); |
670 | if (unlikely(rbi->page == NULL)) { |
671 | rq->stats.rx_buf_alloc_failure++; |
672 | break; |
673 | } |
674 | rbi->dma_addr = dma_map_page( |
675 | &adapter->pdev->dev, |
676 | rbi->page, 0, PAGE_SIZE, |
677 | DMA_FROM_DEVICE); |
678 | if (dma_mapping_error(dev: &adapter->pdev->dev, |
679 | dma_addr: rbi->dma_addr)) { |
680 | put_page(page: rbi->page); |
681 | rbi->page = NULL; |
682 | rq->stats.rx_buf_alloc_failure++; |
683 | break; |
684 | } |
685 | } else { |
686 | /* rx buffers skipped by the device */ |
687 | } |
688 | val = VMXNET3_RXD_BTYPE_BODY << VMXNET3_RXD_BTYPE_SHIFT; |
689 | } |
690 | |
691 | gd->rxd.addr = cpu_to_le64(rbi->dma_addr); |
692 | gd->dword[2] = cpu_to_le32((!ring->gen << VMXNET3_RXD_GEN_SHIFT) |
693 | | val | rbi->len); |
694 | |
695 | /* Fill the last buffer but dont mark it ready, or else the |
696 | * device will think that the queue is full */ |
697 | if (num_allocated == num_to_alloc) { |
698 | rbi->comp_state = VMXNET3_RXD_COMP_DONE; |
699 | break; |
700 | } |
701 | |
702 | gd->dword[2] |= cpu_to_le32(ring->gen << VMXNET3_RXD_GEN_SHIFT); |
703 | num_allocated++; |
704 | vmxnet3_cmd_ring_adv_next2fill(ring); |
705 | } |
706 | |
707 | netdev_dbg(adapter->netdev, |
708 | "alloc_rx_buf: %d allocated, next2fill %u, next2comp %u\n" , |
709 | num_allocated, ring->next2fill, ring->next2comp); |
710 | |
711 | /* so that the device can distinguish a full ring and an empty ring */ |
712 | BUG_ON(num_allocated != 0 && ring->next2fill == ring->next2comp); |
713 | |
714 | return num_allocated; |
715 | } |
716 | |
717 | |
718 | static void |
719 | vmxnet3_append_frag(struct sk_buff *skb, struct Vmxnet3_RxCompDesc *rcd, |
720 | struct vmxnet3_rx_buf_info *rbi) |
721 | { |
722 | skb_frag_t *frag = skb_shinfo(skb)->frags + skb_shinfo(skb)->nr_frags; |
723 | |
724 | BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS); |
725 | |
726 | skb_frag_fill_page_desc(frag, page: rbi->page, off: 0, size: rcd->len); |
727 | skb->data_len += rcd->len; |
728 | skb->truesize += PAGE_SIZE; |
729 | skb_shinfo(skb)->nr_frags++; |
730 | } |
731 | |
732 | |
733 | static int |
734 | vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx, |
735 | struct vmxnet3_tx_queue *tq, struct pci_dev *pdev, |
736 | struct vmxnet3_adapter *adapter) |
737 | { |
738 | u32 dw2, len; |
739 | unsigned long buf_offset; |
740 | int i; |
741 | union Vmxnet3_GenericDesc *gdesc; |
742 | struct vmxnet3_tx_buf_info *tbi = NULL; |
743 | |
744 | BUG_ON(ctx->copy_size > skb_headlen(skb)); |
745 | |
746 | /* use the previous gen bit for the SOP desc */ |
747 | dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT; |
748 | |
749 | ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill; |
750 | gdesc = ctx->sop_txd; /* both loops below can be skipped */ |
751 | |
752 | /* no need to map the buffer if headers are copied */ |
753 | if (ctx->copy_size) { |
754 | ctx->sop_txd->txd.addr = cpu_to_le64(tq->data_ring.basePA + |
755 | tq->tx_ring.next2fill * |
756 | tq->txdata_desc_size); |
757 | ctx->sop_txd->dword[2] = cpu_to_le32(dw2 | ctx->copy_size); |
758 | ctx->sop_txd->dword[3] = 0; |
759 | |
760 | tbi = tq->buf_info + tq->tx_ring.next2fill; |
761 | tbi->map_type = VMXNET3_MAP_NONE; |
762 | |
763 | netdev_dbg(adapter->netdev, |
764 | "txd[%u]: 0x%Lx 0x%x 0x%x\n" , |
765 | tq->tx_ring.next2fill, |
766 | le64_to_cpu(ctx->sop_txd->txd.addr), |
767 | ctx->sop_txd->dword[2], ctx->sop_txd->dword[3]); |
768 | vmxnet3_cmd_ring_adv_next2fill(ring: &tq->tx_ring); |
769 | |
770 | /* use the right gen for non-SOP desc */ |
771 | dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; |
772 | } |
773 | |
774 | /* linear part can use multiple tx desc if it's big */ |
775 | len = skb_headlen(skb) - ctx->copy_size; |
776 | buf_offset = ctx->copy_size; |
777 | while (len) { |
778 | u32 buf_size; |
779 | |
780 | if (len < VMXNET3_MAX_TX_BUF_SIZE) { |
781 | buf_size = len; |
782 | dw2 |= len; |
783 | } else { |
784 | buf_size = VMXNET3_MAX_TX_BUF_SIZE; |
785 | /* spec says that for TxDesc.len, 0 == 2^14 */ |
786 | } |
787 | |
788 | tbi = tq->buf_info + tq->tx_ring.next2fill; |
789 | tbi->map_type = VMXNET3_MAP_SINGLE; |
790 | tbi->dma_addr = dma_map_single(&adapter->pdev->dev, |
791 | skb->data + buf_offset, buf_size, |
792 | DMA_TO_DEVICE); |
793 | if (dma_mapping_error(dev: &adapter->pdev->dev, dma_addr: tbi->dma_addr)) |
794 | return -EFAULT; |
795 | |
796 | tbi->len = buf_size; |
797 | |
798 | gdesc = tq->tx_ring.base + tq->tx_ring.next2fill; |
799 | BUG_ON(gdesc->txd.gen == tq->tx_ring.gen); |
800 | |
801 | gdesc->txd.addr = cpu_to_le64(tbi->dma_addr); |
802 | gdesc->dword[2] = cpu_to_le32(dw2); |
803 | gdesc->dword[3] = 0; |
804 | |
805 | netdev_dbg(adapter->netdev, |
806 | "txd[%u]: 0x%Lx 0x%x 0x%x\n" , |
807 | tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr), |
808 | le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]); |
809 | vmxnet3_cmd_ring_adv_next2fill(ring: &tq->tx_ring); |
810 | dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; |
811 | |
812 | len -= buf_size; |
813 | buf_offset += buf_size; |
814 | } |
815 | |
816 | for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { |
817 | const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; |
818 | u32 buf_size; |
819 | |
820 | buf_offset = 0; |
821 | len = skb_frag_size(frag); |
822 | while (len) { |
823 | tbi = tq->buf_info + tq->tx_ring.next2fill; |
824 | if (len < VMXNET3_MAX_TX_BUF_SIZE) { |
825 | buf_size = len; |
826 | dw2 |= len; |
827 | } else { |
828 | buf_size = VMXNET3_MAX_TX_BUF_SIZE; |
829 | /* spec says that for TxDesc.len, 0 == 2^14 */ |
830 | } |
831 | tbi->map_type = VMXNET3_MAP_PAGE; |
832 | tbi->dma_addr = skb_frag_dma_map(dev: &adapter->pdev->dev, frag, |
833 | offset: buf_offset, size: buf_size, |
834 | dir: DMA_TO_DEVICE); |
835 | if (dma_mapping_error(dev: &adapter->pdev->dev, dma_addr: tbi->dma_addr)) |
836 | return -EFAULT; |
837 | |
838 | tbi->len = buf_size; |
839 | |
840 | gdesc = tq->tx_ring.base + tq->tx_ring.next2fill; |
841 | BUG_ON(gdesc->txd.gen == tq->tx_ring.gen); |
842 | |
843 | gdesc->txd.addr = cpu_to_le64(tbi->dma_addr); |
844 | gdesc->dword[2] = cpu_to_le32(dw2); |
845 | gdesc->dword[3] = 0; |
846 | |
847 | netdev_dbg(adapter->netdev, |
848 | "txd[%u]: 0x%llx %u %u\n" , |
849 | tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr), |
850 | le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]); |
851 | vmxnet3_cmd_ring_adv_next2fill(ring: &tq->tx_ring); |
852 | dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; |
853 | |
854 | len -= buf_size; |
855 | buf_offset += buf_size; |
856 | } |
857 | } |
858 | |
859 | ctx->eop_txd = gdesc; |
860 | |
861 | /* set the last buf_info for the pkt */ |
862 | tbi->skb = skb; |
863 | tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base; |
864 | |
865 | return 0; |
866 | } |
867 | |
868 | |
869 | /* Init all tx queues */ |
870 | static void |
871 | vmxnet3_tq_init_all(struct vmxnet3_adapter *adapter) |
872 | { |
873 | int i; |
874 | |
875 | for (i = 0; i < adapter->num_tx_queues; i++) |
876 | vmxnet3_tq_init(tq: &adapter->tx_queue[i], adapter); |
877 | } |
878 | |
879 | |
880 | /* |
881 | * parse relevant protocol headers: |
882 | * For a tso pkt, relevant headers are L2/3/4 including options |
883 | * For a pkt requesting csum offloading, they are L2/3 and may include L4 |
884 | * if it's a TCP/UDP pkt |
885 | * |
886 | * Returns: |
887 | * -1: error happens during parsing |
888 | * 0: protocol headers parsed, but too big to be copied |
889 | * 1: protocol headers parsed and copied |
890 | * |
891 | * Other effects: |
892 | * 1. related *ctx fields are updated. |
893 | * 2. ctx->copy_size is # of bytes copied |
894 | * 3. the portion to be copied is guaranteed to be in the linear part |
895 | * |
896 | */ |
897 | static int |
898 | vmxnet3_parse_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, |
899 | struct vmxnet3_tx_ctx *ctx, |
900 | struct vmxnet3_adapter *adapter) |
901 | { |
902 | u8 protocol = 0; |
903 | |
904 | if (ctx->mss) { /* TSO */ |
905 | if (VMXNET3_VERSION_GE_4(adapter) && skb->encapsulation) { |
906 | ctx->l4_offset = skb_inner_transport_offset(skb); |
907 | ctx->l4_hdr_size = inner_tcp_hdrlen(skb); |
908 | ctx->copy_size = ctx->l4_offset + ctx->l4_hdr_size; |
909 | } else { |
910 | ctx->l4_offset = skb_transport_offset(skb); |
911 | ctx->l4_hdr_size = tcp_hdrlen(skb); |
912 | ctx->copy_size = ctx->l4_offset + ctx->l4_hdr_size; |
913 | } |
914 | } else { |
915 | if (skb->ip_summed == CHECKSUM_PARTIAL) { |
916 | /* For encap packets, skb_checksum_start_offset refers |
917 | * to inner L4 offset. Thus, below works for encap as |
918 | * well as non-encap case |
919 | */ |
920 | ctx->l4_offset = skb_checksum_start_offset(skb); |
921 | |
922 | if (VMXNET3_VERSION_GE_4(adapter) && |
923 | skb->encapsulation) { |
924 | struct iphdr *iph = inner_ip_hdr(skb); |
925 | |
926 | if (iph->version == 4) { |
927 | protocol = iph->protocol; |
928 | } else { |
929 | const struct ipv6hdr *ipv6h; |
930 | |
931 | ipv6h = inner_ipv6_hdr(skb); |
932 | protocol = ipv6h->nexthdr; |
933 | } |
934 | } else { |
935 | if (ctx->ipv4) { |
936 | const struct iphdr *iph = ip_hdr(skb); |
937 | |
938 | protocol = iph->protocol; |
939 | } else if (ctx->ipv6) { |
940 | const struct ipv6hdr *ipv6h; |
941 | |
942 | ipv6h = ipv6_hdr(skb); |
943 | protocol = ipv6h->nexthdr; |
944 | } |
945 | } |
946 | |
947 | switch (protocol) { |
948 | case IPPROTO_TCP: |
949 | ctx->l4_hdr_size = skb->encapsulation ? inner_tcp_hdrlen(skb) : |
950 | tcp_hdrlen(skb); |
951 | break; |
952 | case IPPROTO_UDP: |
953 | ctx->l4_hdr_size = sizeof(struct udphdr); |
954 | break; |
955 | default: |
956 | ctx->l4_hdr_size = 0; |
957 | break; |
958 | } |
959 | |
960 | ctx->copy_size = min(ctx->l4_offset + |
961 | ctx->l4_hdr_size, skb->len); |
962 | } else { |
963 | ctx->l4_offset = 0; |
964 | ctx->l4_hdr_size = 0; |
965 | /* copy as much as allowed */ |
966 | ctx->copy_size = min_t(unsigned int, |
967 | tq->txdata_desc_size, |
968 | skb_headlen(skb)); |
969 | } |
970 | |
971 | if (skb->len <= VMXNET3_HDR_COPY_SIZE) |
972 | ctx->copy_size = skb->len; |
973 | |
974 | /* make sure headers are accessible directly */ |
975 | if (unlikely(!pskb_may_pull(skb, ctx->copy_size))) |
976 | goto err; |
977 | } |
978 | |
979 | if (unlikely(ctx->copy_size > tq->txdata_desc_size)) { |
980 | tq->stats.oversized_hdr++; |
981 | ctx->copy_size = 0; |
982 | return 0; |
983 | } |
984 | |
985 | return 1; |
986 | err: |
987 | return -1; |
988 | } |
989 | |
990 | /* |
991 | * copy relevant protocol headers to the transmit ring: |
992 | * For a tso pkt, relevant headers are L2/3/4 including options |
993 | * For a pkt requesting csum offloading, they are L2/3 and may include L4 |
994 | * if it's a TCP/UDP pkt |
995 | * |
996 | * |
997 | * Note that this requires that vmxnet3_parse_hdr be called first to set the |
998 | * appropriate bits in ctx first |
999 | */ |
1000 | static void |
1001 | vmxnet3_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, |
1002 | struct vmxnet3_tx_ctx *ctx, |
1003 | struct vmxnet3_adapter *adapter) |
1004 | { |
1005 | struct Vmxnet3_TxDataDesc *tdd; |
1006 | |
1007 | tdd = (struct Vmxnet3_TxDataDesc *)((u8 *)tq->data_ring.base + |
1008 | tq->tx_ring.next2fill * |
1009 | tq->txdata_desc_size); |
1010 | |
1011 | memcpy(tdd->data, skb->data, ctx->copy_size); |
1012 | netdev_dbg(adapter->netdev, |
1013 | "copy %u bytes to dataRing[%u]\n" , |
1014 | ctx->copy_size, tq->tx_ring.next2fill); |
1015 | } |
1016 | |
1017 | |
1018 | static void |
1019 | vmxnet3_prepare_inner_tso(struct sk_buff *skb, |
1020 | struct vmxnet3_tx_ctx *ctx) |
1021 | { |
1022 | struct tcphdr *tcph = inner_tcp_hdr(skb); |
1023 | struct iphdr *iph = inner_ip_hdr(skb); |
1024 | |
1025 | if (iph->version == 4) { |
1026 | iph->check = 0; |
1027 | tcph->check = ~csum_tcpudp_magic(saddr: iph->saddr, daddr: iph->daddr, len: 0, |
1028 | IPPROTO_TCP, sum: 0); |
1029 | } else { |
1030 | struct ipv6hdr *iph = inner_ipv6_hdr(skb); |
1031 | |
1032 | tcph->check = ~csum_ipv6_magic(saddr: &iph->saddr, daddr: &iph->daddr, len: 0, |
1033 | IPPROTO_TCP, sum: 0); |
1034 | } |
1035 | } |
1036 | |
1037 | static void |
1038 | vmxnet3_prepare_tso(struct sk_buff *skb, |
1039 | struct vmxnet3_tx_ctx *ctx) |
1040 | { |
1041 | struct tcphdr *tcph = tcp_hdr(skb); |
1042 | |
1043 | if (ctx->ipv4) { |
1044 | struct iphdr *iph = ip_hdr(skb); |
1045 | |
1046 | iph->check = 0; |
1047 | tcph->check = ~csum_tcpudp_magic(saddr: iph->saddr, daddr: iph->daddr, len: 0, |
1048 | IPPROTO_TCP, sum: 0); |
1049 | } else if (ctx->ipv6) { |
1050 | tcp_v6_gso_csum_prep(skb); |
1051 | } |
1052 | } |
1053 | |
1054 | static int txd_estimate(const struct sk_buff *skb) |
1055 | { |
1056 | int count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1; |
1057 | int i; |
1058 | |
1059 | for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { |
1060 | const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; |
1061 | |
1062 | count += VMXNET3_TXD_NEEDED(skb_frag_size(frag)); |
1063 | } |
1064 | return count; |
1065 | } |
1066 | |
1067 | /* |
1068 | * Transmits a pkt thru a given tq |
1069 | * Returns: |
1070 | * NETDEV_TX_OK: descriptors are setup successfully |
1071 | * NETDEV_TX_OK: error occurred, the pkt is dropped |
1072 | * NETDEV_TX_BUSY: tx ring is full, queue is stopped |
1073 | * |
1074 | * Side-effects: |
1075 | * 1. tx ring may be changed |
1076 | * 2. tq stats may be updated accordingly |
1077 | * 3. shared->txNumDeferred may be updated |
1078 | */ |
1079 | |
1080 | static int |
1081 | vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, |
1082 | struct vmxnet3_adapter *adapter, struct net_device *netdev) |
1083 | { |
1084 | int ret; |
1085 | u32 count; |
1086 | int num_pkts; |
1087 | int tx_num_deferred; |
1088 | unsigned long flags; |
1089 | struct vmxnet3_tx_ctx ctx; |
1090 | union Vmxnet3_GenericDesc *gdesc; |
1091 | #ifdef __BIG_ENDIAN_BITFIELD |
1092 | /* Use temporary descriptor to avoid touching bits multiple times */ |
1093 | union Vmxnet3_GenericDesc tempTxDesc; |
1094 | #endif |
1095 | |
1096 | count = txd_estimate(skb); |
1097 | |
1098 | ctx.ipv4 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IP)); |
1099 | ctx.ipv6 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IPV6)); |
1100 | |
1101 | ctx.mss = skb_shinfo(skb)->gso_size; |
1102 | if (ctx.mss) { |
1103 | if (skb_header_cloned(skb)) { |
1104 | if (unlikely(pskb_expand_head(skb, 0, 0, |
1105 | GFP_ATOMIC) != 0)) { |
1106 | tq->stats.drop_tso++; |
1107 | goto drop_pkt; |
1108 | } |
1109 | tq->stats.copy_skb_header++; |
1110 | } |
1111 | if (unlikely(count > VMXNET3_MAX_TSO_TXD_PER_PKT)) { |
1112 | /* tso pkts must not use more than |
1113 | * VMXNET3_MAX_TSO_TXD_PER_PKT entries |
1114 | */ |
1115 | if (skb_linearize(skb) != 0) { |
1116 | tq->stats.drop_too_many_frags++; |
1117 | goto drop_pkt; |
1118 | } |
1119 | tq->stats.linearized++; |
1120 | |
1121 | /* recalculate the # of descriptors to use */ |
1122 | count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1; |
1123 | if (unlikely(count > VMXNET3_MAX_TSO_TXD_PER_PKT)) { |
1124 | tq->stats.drop_too_many_frags++; |
1125 | goto drop_pkt; |
1126 | } |
1127 | } |
1128 | if (skb->encapsulation) { |
1129 | vmxnet3_prepare_inner_tso(skb, ctx: &ctx); |
1130 | } else { |
1131 | vmxnet3_prepare_tso(skb, ctx: &ctx); |
1132 | } |
1133 | } else { |
1134 | if (unlikely(count > VMXNET3_MAX_TXD_PER_PKT)) { |
1135 | |
1136 | /* non-tso pkts must not use more than |
1137 | * VMXNET3_MAX_TXD_PER_PKT entries |
1138 | */ |
1139 | if (skb_linearize(skb) != 0) { |
1140 | tq->stats.drop_too_many_frags++; |
1141 | goto drop_pkt; |
1142 | } |
1143 | tq->stats.linearized++; |
1144 | |
1145 | /* recalculate the # of descriptors to use */ |
1146 | count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1; |
1147 | } |
1148 | } |
1149 | |
1150 | ret = vmxnet3_parse_hdr(skb, tq, ctx: &ctx, adapter); |
1151 | if (ret >= 0) { |
1152 | BUG_ON(ret <= 0 && ctx.copy_size != 0); |
1153 | /* hdrs parsed, check against other limits */ |
1154 | if (ctx.mss) { |
1155 | if (unlikely(ctx.l4_offset + ctx.l4_hdr_size > |
1156 | VMXNET3_MAX_TX_BUF_SIZE)) { |
1157 | tq->stats.drop_oversized_hdr++; |
1158 | goto drop_pkt; |
1159 | } |
1160 | } else { |
1161 | if (skb->ip_summed == CHECKSUM_PARTIAL) { |
1162 | if (unlikely(ctx.l4_offset + |
1163 | skb->csum_offset > |
1164 | VMXNET3_MAX_CSUM_OFFSET)) { |
1165 | tq->stats.drop_oversized_hdr++; |
1166 | goto drop_pkt; |
1167 | } |
1168 | } |
1169 | } |
1170 | } else { |
1171 | tq->stats.drop_hdr_inspect_err++; |
1172 | goto drop_pkt; |
1173 | } |
1174 | |
1175 | spin_lock_irqsave(&tq->tx_lock, flags); |
1176 | |
1177 | if (count > vmxnet3_cmd_ring_desc_avail(ring: &tq->tx_ring)) { |
1178 | tq->stats.tx_ring_full++; |
1179 | netdev_dbg(adapter->netdev, |
1180 | "tx queue stopped on %s, next2comp %u" |
1181 | " next2fill %u\n" , adapter->netdev->name, |
1182 | tq->tx_ring.next2comp, tq->tx_ring.next2fill); |
1183 | |
1184 | vmxnet3_tq_stop(tq, adapter); |
1185 | spin_unlock_irqrestore(lock: &tq->tx_lock, flags); |
1186 | return NETDEV_TX_BUSY; |
1187 | } |
1188 | |
1189 | |
1190 | vmxnet3_copy_hdr(skb, tq, ctx: &ctx, adapter); |
1191 | |
1192 | /* fill tx descs related to addr & len */ |
1193 | if (vmxnet3_map_pkt(skb, ctx: &ctx, tq, pdev: adapter->pdev, adapter)) |
1194 | goto unlock_drop_pkt; |
1195 | |
1196 | /* setup the EOP desc */ |
1197 | ctx.eop_txd->dword[3] = cpu_to_le32(VMXNET3_TXD_CQ | VMXNET3_TXD_EOP); |
1198 | |
1199 | /* setup the SOP desc */ |
1200 | #ifdef __BIG_ENDIAN_BITFIELD |
1201 | gdesc = &tempTxDesc; |
1202 | gdesc->dword[2] = ctx.sop_txd->dword[2]; |
1203 | gdesc->dword[3] = ctx.sop_txd->dword[3]; |
1204 | #else |
1205 | gdesc = ctx.sop_txd; |
1206 | #endif |
1207 | tx_num_deferred = le32_to_cpu(tq->shared->txNumDeferred); |
1208 | if (ctx.mss) { |
1209 | if (VMXNET3_VERSION_GE_4(adapter) && skb->encapsulation) { |
1210 | gdesc->txd.hlen = ctx.l4_offset + ctx.l4_hdr_size; |
1211 | if (VMXNET3_VERSION_GE_7(adapter)) { |
1212 | gdesc->txd.om = VMXNET3_OM_TSO; |
1213 | gdesc->txd.ext1 = 1; |
1214 | } else { |
1215 | gdesc->txd.om = VMXNET3_OM_ENCAP; |
1216 | } |
1217 | gdesc->txd.msscof = ctx.mss; |
1218 | |
1219 | if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM) |
1220 | gdesc->txd.oco = 1; |
1221 | } else { |
1222 | gdesc->txd.hlen = ctx.l4_offset + ctx.l4_hdr_size; |
1223 | gdesc->txd.om = VMXNET3_OM_TSO; |
1224 | gdesc->txd.msscof = ctx.mss; |
1225 | } |
1226 | num_pkts = (skb->len - gdesc->txd.hlen + ctx.mss - 1) / ctx.mss; |
1227 | } else { |
1228 | if (skb->ip_summed == CHECKSUM_PARTIAL) { |
1229 | if (VMXNET3_VERSION_GE_4(adapter) && |
1230 | skb->encapsulation) { |
1231 | gdesc->txd.hlen = ctx.l4_offset + |
1232 | ctx.l4_hdr_size; |
1233 | if (VMXNET3_VERSION_GE_7(adapter)) { |
1234 | gdesc->txd.om = VMXNET3_OM_CSUM; |
1235 | gdesc->txd.msscof = ctx.l4_offset + |
1236 | skb->csum_offset; |
1237 | gdesc->txd.ext1 = 1; |
1238 | } else { |
1239 | gdesc->txd.om = VMXNET3_OM_ENCAP; |
1240 | gdesc->txd.msscof = 0; /* Reserved */ |
1241 | } |
1242 | } else { |
1243 | gdesc->txd.hlen = ctx.l4_offset; |
1244 | gdesc->txd.om = VMXNET3_OM_CSUM; |
1245 | gdesc->txd.msscof = ctx.l4_offset + |
1246 | skb->csum_offset; |
1247 | } |
1248 | } else { |
1249 | gdesc->txd.om = 0; |
1250 | gdesc->txd.msscof = 0; |
1251 | } |
1252 | num_pkts = 1; |
1253 | } |
1254 | le32_add_cpu(var: &tq->shared->txNumDeferred, val: num_pkts); |
1255 | tx_num_deferred += num_pkts; |
1256 | |
1257 | if (skb_vlan_tag_present(skb)) { |
1258 | gdesc->txd.ti = 1; |
1259 | gdesc->txd.tci = skb_vlan_tag_get(skb); |
1260 | } |
1261 | |
1262 | /* Ensure that the write to (&gdesc->txd)->gen will be observed after |
1263 | * all other writes to &gdesc->txd. |
1264 | */ |
1265 | dma_wmb(); |
1266 | |
1267 | /* finally flips the GEN bit of the SOP desc. */ |
1268 | gdesc->dword[2] = cpu_to_le32(le32_to_cpu(gdesc->dword[2]) ^ |
1269 | VMXNET3_TXD_GEN); |
1270 | #ifdef __BIG_ENDIAN_BITFIELD |
1271 | /* Finished updating in bitfields of Tx Desc, so write them in original |
1272 | * place. |
1273 | */ |
1274 | vmxnet3_TxDescToLe((struct Vmxnet3_TxDesc *)gdesc, |
1275 | (struct Vmxnet3_TxDesc *)ctx.sop_txd); |
1276 | gdesc = ctx.sop_txd; |
1277 | #endif |
1278 | netdev_dbg(adapter->netdev, |
1279 | "txd[%u]: SOP 0x%Lx 0x%x 0x%x\n" , |
1280 | (u32)(ctx.sop_txd - |
1281 | tq->tx_ring.base), le64_to_cpu(gdesc->txd.addr), |
1282 | le32_to_cpu(gdesc->dword[2]), le32_to_cpu(gdesc->dword[3])); |
1283 | |
1284 | spin_unlock_irqrestore(lock: &tq->tx_lock, flags); |
1285 | |
1286 | if (tx_num_deferred >= le32_to_cpu(tq->shared->txThreshold)) { |
1287 | tq->shared->txNumDeferred = 0; |
1288 | VMXNET3_WRITE_BAR0_REG(adapter, |
1289 | adapter->tx_prod_offset + tq->qid * 8, |
1290 | tq->tx_ring.next2fill); |
1291 | } |
1292 | |
1293 | return NETDEV_TX_OK; |
1294 | |
1295 | unlock_drop_pkt: |
1296 | spin_unlock_irqrestore(lock: &tq->tx_lock, flags); |
1297 | drop_pkt: |
1298 | tq->stats.drop_total++; |
1299 | dev_kfree_skb_any(skb); |
1300 | return NETDEV_TX_OK; |
1301 | } |
1302 | |
1303 | static int |
1304 | vmxnet3_create_pp(struct vmxnet3_adapter *adapter, |
1305 | struct vmxnet3_rx_queue *rq, int size) |
1306 | { |
1307 | bool xdp_prog = vmxnet3_xdp_enabled(adapter); |
1308 | const struct page_pool_params pp_params = { |
1309 | .order = 0, |
1310 | .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, |
1311 | .pool_size = size, |
1312 | .nid = NUMA_NO_NODE, |
1313 | .dev = &adapter->pdev->dev, |
1314 | .offset = VMXNET3_XDP_RX_OFFSET, |
1315 | .max_len = VMXNET3_XDP_MAX_FRSIZE, |
1316 | .dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE, |
1317 | }; |
1318 | struct page_pool *pp; |
1319 | int err; |
1320 | |
1321 | pp = page_pool_create(params: &pp_params); |
1322 | if (IS_ERR(ptr: pp)) |
1323 | return PTR_ERR(ptr: pp); |
1324 | |
1325 | err = xdp_rxq_info_reg(xdp_rxq: &rq->xdp_rxq, dev: adapter->netdev, queue_index: rq->qid, |
1326 | napi_id: rq->napi.napi_id); |
1327 | if (err < 0) |
1328 | goto err_free_pp; |
1329 | |
1330 | err = xdp_rxq_info_reg_mem_model(xdp_rxq: &rq->xdp_rxq, type: MEM_TYPE_PAGE_POOL, allocator: pp); |
1331 | if (err) |
1332 | goto err_unregister_rxq; |
1333 | |
1334 | rq->page_pool = pp; |
1335 | |
1336 | return 0; |
1337 | |
1338 | err_unregister_rxq: |
1339 | xdp_rxq_info_unreg(xdp_rxq: &rq->xdp_rxq); |
1340 | err_free_pp: |
1341 | page_pool_destroy(pool: pp); |
1342 | |
1343 | return err; |
1344 | } |
1345 | |
1346 | void * |
1347 | vmxnet3_pp_get_buff(struct page_pool *pp, dma_addr_t *dma_addr, |
1348 | gfp_t gfp_mask) |
1349 | { |
1350 | struct page *page; |
1351 | |
1352 | page = page_pool_alloc_pages(pool: pp, gfp: gfp_mask | __GFP_NOWARN); |
1353 | if (unlikely(!page)) |
1354 | return NULL; |
1355 | |
1356 | *dma_addr = page_pool_get_dma_addr(page) + pp->p.offset; |
1357 | |
1358 | return page_address(page); |
1359 | } |
1360 | |
1361 | static netdev_tx_t |
1362 | vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev) |
1363 | { |
1364 | struct vmxnet3_adapter *adapter = netdev_priv(dev: netdev); |
1365 | |
1366 | BUG_ON(skb->queue_mapping > adapter->num_tx_queues); |
1367 | return vmxnet3_tq_xmit(skb, |
1368 | tq: &adapter->tx_queue[skb->queue_mapping], |
1369 | adapter, netdev); |
1370 | } |
1371 | |
1372 | |
1373 | static void |
1374 | vmxnet3_rx_csum(struct vmxnet3_adapter *adapter, |
1375 | struct sk_buff *skb, |
1376 | union Vmxnet3_GenericDesc *gdesc) |
1377 | { |
1378 | if (!gdesc->rcd.cnc && adapter->netdev->features & NETIF_F_RXCSUM) { |
1379 | if (gdesc->rcd.v4 && |
1380 | (le32_to_cpu(gdesc->dword[3]) & |
1381 | VMXNET3_RCD_CSUM_OK) == VMXNET3_RCD_CSUM_OK) { |
1382 | skb->ip_summed = CHECKSUM_UNNECESSARY; |
1383 | if ((le32_to_cpu(gdesc->dword[0]) & |
1384 | (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))) { |
1385 | skb->csum_level = 1; |
1386 | } |
1387 | WARN_ON_ONCE(!(gdesc->rcd.tcp || gdesc->rcd.udp) && |
1388 | !(le32_to_cpu(gdesc->dword[0]) & |
1389 | (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); |
1390 | WARN_ON_ONCE(gdesc->rcd.frg && |
1391 | !(le32_to_cpu(gdesc->dword[0]) & |
1392 | (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); |
1393 | } else if (gdesc->rcd.v6 && (le32_to_cpu(gdesc->dword[3]) & |
1394 | (1 << VMXNET3_RCD_TUC_SHIFT))) { |
1395 | skb->ip_summed = CHECKSUM_UNNECESSARY; |
1396 | if ((le32_to_cpu(gdesc->dword[0]) & |
1397 | (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))) { |
1398 | skb->csum_level = 1; |
1399 | } |
1400 | WARN_ON_ONCE(!(gdesc->rcd.tcp || gdesc->rcd.udp) && |
1401 | !(le32_to_cpu(gdesc->dword[0]) & |
1402 | (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); |
1403 | WARN_ON_ONCE(gdesc->rcd.frg && |
1404 | !(le32_to_cpu(gdesc->dword[0]) & |
1405 | (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); |
1406 | } else { |
1407 | if (gdesc->rcd.csum) { |
1408 | skb->csum = htons(gdesc->rcd.csum); |
1409 | skb->ip_summed = CHECKSUM_PARTIAL; |
1410 | } else { |
1411 | skb_checksum_none_assert(skb); |
1412 | } |
1413 | } |
1414 | } else { |
1415 | skb_checksum_none_assert(skb); |
1416 | } |
1417 | } |
1418 | |
1419 | |
1420 | static void |
1421 | vmxnet3_rx_error(struct vmxnet3_rx_queue *rq, struct Vmxnet3_RxCompDesc *rcd, |
1422 | struct vmxnet3_rx_ctx *ctx, struct vmxnet3_adapter *adapter) |
1423 | { |
1424 | rq->stats.drop_err++; |
1425 | if (!rcd->fcs) |
1426 | rq->stats.drop_fcs++; |
1427 | |
1428 | rq->stats.drop_total++; |
1429 | |
1430 | /* |
1431 | * We do not unmap and chain the rx buffer to the skb. |
1432 | * We basically pretend this buffer is not used and will be recycled |
1433 | * by vmxnet3_rq_alloc_rx_buf() |
1434 | */ |
1435 | |
1436 | /* |
1437 | * ctx->skb may be NULL if this is the first and the only one |
1438 | * desc for the pkt |
1439 | */ |
1440 | if (ctx->skb) |
1441 | dev_kfree_skb_irq(skb: ctx->skb); |
1442 | |
1443 | ctx->skb = NULL; |
1444 | } |
1445 | |
1446 | |
1447 | static u32 |
1448 | vmxnet3_get_hdr_len(struct vmxnet3_adapter *adapter, struct sk_buff *skb, |
1449 | union Vmxnet3_GenericDesc *gdesc) |
1450 | { |
1451 | u32 hlen, maplen; |
1452 | union { |
1453 | void *ptr; |
1454 | struct ethhdr *eth; |
1455 | struct vlan_ethhdr *veth; |
1456 | struct iphdr *ipv4; |
1457 | struct ipv6hdr *ipv6; |
1458 | struct tcphdr *tcp; |
1459 | } hdr; |
1460 | BUG_ON(gdesc->rcd.tcp == 0); |
1461 | |
1462 | maplen = skb_headlen(skb); |
1463 | if (unlikely(sizeof(struct iphdr) + sizeof(struct tcphdr) > maplen)) |
1464 | return 0; |
1465 | |
1466 | if (skb->protocol == cpu_to_be16(ETH_P_8021Q) || |
1467 | skb->protocol == cpu_to_be16(ETH_P_8021AD)) |
1468 | hlen = sizeof(struct vlan_ethhdr); |
1469 | else |
1470 | hlen = sizeof(struct ethhdr); |
1471 | |
1472 | hdr.eth = eth_hdr(skb); |
1473 | if (gdesc->rcd.v4) { |
1474 | BUG_ON(hdr.eth->h_proto != htons(ETH_P_IP) && |
1475 | hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IP)); |
1476 | hdr.ptr += hlen; |
1477 | BUG_ON(hdr.ipv4->protocol != IPPROTO_TCP); |
1478 | hlen = hdr.ipv4->ihl << 2; |
1479 | hdr.ptr += hdr.ipv4->ihl << 2; |
1480 | } else if (gdesc->rcd.v6) { |
1481 | BUG_ON(hdr.eth->h_proto != htons(ETH_P_IPV6) && |
1482 | hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IPV6)); |
1483 | hdr.ptr += hlen; |
1484 | /* Use an estimated value, since we also need to handle |
1485 | * TSO case. |
1486 | */ |
1487 | if (hdr.ipv6->nexthdr != IPPROTO_TCP) |
1488 | return sizeof(struct ipv6hdr) + sizeof(struct tcphdr); |
1489 | hlen = sizeof(struct ipv6hdr); |
1490 | hdr.ptr += sizeof(struct ipv6hdr); |
1491 | } else { |
1492 | /* Non-IP pkt, dont estimate header length */ |
1493 | return 0; |
1494 | } |
1495 | |
1496 | if (hlen + sizeof(struct tcphdr) > maplen) |
1497 | return 0; |
1498 | |
1499 | return (hlen + (hdr.tcp->doff << 2)); |
1500 | } |
1501 | |
1502 | static int |
1503 | vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq, |
1504 | struct vmxnet3_adapter *adapter, int quota) |
1505 | { |
1506 | u32 rxprod_reg[2] = { |
1507 | adapter->rx_prod_offset, adapter->rx_prod2_offset |
1508 | }; |
1509 | u32 num_pkts = 0; |
1510 | bool skip_page_frags = false; |
1511 | bool encap_lro = false; |
1512 | struct Vmxnet3_RxCompDesc *rcd; |
1513 | struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx; |
1514 | u16 segCnt = 0, mss = 0; |
1515 | int comp_offset, fill_offset; |
1516 | #ifdef __BIG_ENDIAN_BITFIELD |
1517 | struct Vmxnet3_RxDesc rxCmdDesc; |
1518 | struct Vmxnet3_RxCompDesc rxComp; |
1519 | #endif |
1520 | bool need_flush = false; |
1521 | |
1522 | vmxnet3_getRxComp(rcd, &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, |
1523 | &rxComp); |
1524 | while (rcd->gen == rq->comp_ring.gen) { |
1525 | struct vmxnet3_rx_buf_info *rbi; |
1526 | struct sk_buff *skb, *new_skb = NULL; |
1527 | struct page *new_page = NULL; |
1528 | dma_addr_t new_dma_addr; |
1529 | int num_to_alloc; |
1530 | struct Vmxnet3_RxDesc *rxd; |
1531 | u32 idx, ring_idx; |
1532 | struct vmxnet3_cmd_ring *ring = NULL; |
1533 | if (num_pkts >= quota) { |
1534 | /* we may stop even before we see the EOP desc of |
1535 | * the current pkt |
1536 | */ |
1537 | break; |
1538 | } |
1539 | |
1540 | /* Prevent any rcd field from being (speculatively) read before |
1541 | * rcd->gen is read. |
1542 | */ |
1543 | dma_rmb(); |
1544 | |
1545 | BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2 && |
1546 | rcd->rqID != rq->dataRingQid); |
1547 | idx = rcd->rxdIdx; |
1548 | ring_idx = VMXNET3_GET_RING_IDX(adapter, rcd->rqID); |
1549 | ring = rq->rx_ring + ring_idx; |
1550 | vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd, |
1551 | &rxCmdDesc); |
1552 | rbi = rq->buf_info[ring_idx] + idx; |
1553 | |
1554 | BUG_ON(rxd->addr != rbi->dma_addr || |
1555 | rxd->len != rbi->len); |
1556 | |
1557 | if (unlikely(rcd->eop && rcd->err)) { |
1558 | vmxnet3_rx_error(rq, rcd, ctx, adapter); |
1559 | goto rcd_done; |
1560 | } |
1561 | |
1562 | if (rcd->sop && rcd->eop && vmxnet3_xdp_enabled(adapter)) { |
1563 | struct sk_buff *skb_xdp_pass; |
1564 | int act; |
1565 | |
1566 | if (VMXNET3_RX_DATA_RING(adapter, rcd->rqID)) { |
1567 | ctx->skb = NULL; |
1568 | goto skip_xdp; /* Handle it later. */ |
1569 | } |
1570 | |
1571 | if (rbi->buf_type != VMXNET3_RX_BUF_XDP) |
1572 | goto rcd_done; |
1573 | |
1574 | act = vmxnet3_process_xdp(adapter, rq, rcd, rbi, rxd, |
1575 | skb_xdp_pass: &skb_xdp_pass); |
1576 | if (act == XDP_PASS) { |
1577 | ctx->skb = skb_xdp_pass; |
1578 | goto sop_done; |
1579 | } |
1580 | ctx->skb = NULL; |
1581 | need_flush |= act == XDP_REDIRECT; |
1582 | |
1583 | goto rcd_done; |
1584 | } |
1585 | skip_xdp: |
1586 | |
1587 | if (rcd->sop) { /* first buf of the pkt */ |
1588 | bool rxDataRingUsed; |
1589 | u16 len; |
1590 | |
1591 | BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD || |
1592 | (rcd->rqID != rq->qid && |
1593 | rcd->rqID != rq->dataRingQid)); |
1594 | |
1595 | BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB && |
1596 | rbi->buf_type != VMXNET3_RX_BUF_XDP); |
1597 | BUG_ON(ctx->skb != NULL || rbi->skb == NULL); |
1598 | |
1599 | if (unlikely(rcd->len == 0)) { |
1600 | /* Pretend the rx buffer is skipped. */ |
1601 | BUG_ON(!(rcd->sop && rcd->eop)); |
1602 | netdev_dbg(adapter->netdev, |
1603 | "rxRing[%u][%u] 0 length\n" , |
1604 | ring_idx, idx); |
1605 | goto rcd_done; |
1606 | } |
1607 | |
1608 | skip_page_frags = false; |
1609 | ctx->skb = rbi->skb; |
1610 | |
1611 | rxDataRingUsed = |
1612 | VMXNET3_RX_DATA_RING(adapter, rcd->rqID); |
1613 | len = rxDataRingUsed ? rcd->len : rbi->len; |
1614 | |
1615 | if (rxDataRingUsed && vmxnet3_xdp_enabled(adapter)) { |
1616 | struct sk_buff *skb_xdp_pass; |
1617 | size_t sz; |
1618 | int act; |
1619 | |
1620 | sz = rcd->rxdIdx * rq->data_ring.desc_size; |
1621 | act = vmxnet3_process_xdp_small(adapter, rq, |
1622 | data: &rq->data_ring.base[sz], |
1623 | len: rcd->len, |
1624 | skb_xdp_pass: &skb_xdp_pass); |
1625 | if (act == XDP_PASS) { |
1626 | ctx->skb = skb_xdp_pass; |
1627 | goto sop_done; |
1628 | } |
1629 | need_flush |= act == XDP_REDIRECT; |
1630 | |
1631 | goto rcd_done; |
1632 | } |
1633 | new_skb = netdev_alloc_skb_ip_align(dev: adapter->netdev, |
1634 | length: len); |
1635 | if (new_skb == NULL) { |
1636 | /* Skb allocation failed, do not handover this |
1637 | * skb to stack. Reuse it. Drop the existing pkt |
1638 | */ |
1639 | rq->stats.rx_buf_alloc_failure++; |
1640 | ctx->skb = NULL; |
1641 | rq->stats.drop_total++; |
1642 | skip_page_frags = true; |
1643 | goto rcd_done; |
1644 | } |
1645 | |
1646 | if (rxDataRingUsed && adapter->rxdataring_enabled) { |
1647 | size_t sz; |
1648 | |
1649 | BUG_ON(rcd->len > rq->data_ring.desc_size); |
1650 | |
1651 | ctx->skb = new_skb; |
1652 | sz = rcd->rxdIdx * rq->data_ring.desc_size; |
1653 | memcpy(new_skb->data, |
1654 | &rq->data_ring.base[sz], rcd->len); |
1655 | } else { |
1656 | ctx->skb = rbi->skb; |
1657 | |
1658 | new_dma_addr = |
1659 | dma_map_single(&adapter->pdev->dev, |
1660 | new_skb->data, rbi->len, |
1661 | DMA_FROM_DEVICE); |
1662 | if (dma_mapping_error(dev: &adapter->pdev->dev, |
1663 | dma_addr: new_dma_addr)) { |
1664 | dev_kfree_skb(new_skb); |
1665 | /* Skb allocation failed, do not |
1666 | * handover this skb to stack. Reuse |
1667 | * it. Drop the existing pkt. |
1668 | */ |
1669 | rq->stats.rx_buf_alloc_failure++; |
1670 | ctx->skb = NULL; |
1671 | rq->stats.drop_total++; |
1672 | skip_page_frags = true; |
1673 | goto rcd_done; |
1674 | } |
1675 | |
1676 | dma_unmap_single(&adapter->pdev->dev, |
1677 | rbi->dma_addr, |
1678 | rbi->len, |
1679 | DMA_FROM_DEVICE); |
1680 | |
1681 | /* Immediate refill */ |
1682 | rbi->skb = new_skb; |
1683 | rbi->dma_addr = new_dma_addr; |
1684 | rxd->addr = cpu_to_le64(rbi->dma_addr); |
1685 | rxd->len = rbi->len; |
1686 | } |
1687 | |
1688 | skb_record_rx_queue(skb: ctx->skb, rx_queue: rq->qid); |
1689 | skb_put(skb: ctx->skb, len: rcd->len); |
1690 | |
1691 | if (VMXNET3_VERSION_GE_2(adapter) && |
1692 | rcd->type == VMXNET3_CDTYPE_RXCOMP_LRO) { |
1693 | struct Vmxnet3_RxCompDescExt *rcdlro; |
1694 | union Vmxnet3_GenericDesc *gdesc; |
1695 | |
1696 | rcdlro = (struct Vmxnet3_RxCompDescExt *)rcd; |
1697 | gdesc = (union Vmxnet3_GenericDesc *)rcd; |
1698 | |
1699 | segCnt = rcdlro->segCnt; |
1700 | WARN_ON_ONCE(segCnt == 0); |
1701 | mss = rcdlro->mss; |
1702 | if (unlikely(segCnt <= 1)) |
1703 | segCnt = 0; |
1704 | encap_lro = (le32_to_cpu(gdesc->dword[0]) & |
1705 | (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)); |
1706 | } else { |
1707 | segCnt = 0; |
1708 | } |
1709 | } else { |
1710 | BUG_ON(ctx->skb == NULL && !skip_page_frags); |
1711 | |
1712 | /* non SOP buffer must be type 1 in most cases */ |
1713 | BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE); |
1714 | BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_BODY); |
1715 | |
1716 | /* If an sop buffer was dropped, skip all |
1717 | * following non-sop fragments. They will be reused. |
1718 | */ |
1719 | if (skip_page_frags) |
1720 | goto rcd_done; |
1721 | |
1722 | if (rcd->len) { |
1723 | new_page = alloc_page(GFP_ATOMIC); |
1724 | /* Replacement page frag could not be allocated. |
1725 | * Reuse this page. Drop the pkt and free the |
1726 | * skb which contained this page as a frag. Skip |
1727 | * processing all the following non-sop frags. |
1728 | */ |
1729 | if (unlikely(!new_page)) { |
1730 | rq->stats.rx_buf_alloc_failure++; |
1731 | dev_kfree_skb(ctx->skb); |
1732 | ctx->skb = NULL; |
1733 | skip_page_frags = true; |
1734 | goto rcd_done; |
1735 | } |
1736 | new_dma_addr = dma_map_page(&adapter->pdev->dev, |
1737 | new_page, |
1738 | 0, PAGE_SIZE, |
1739 | DMA_FROM_DEVICE); |
1740 | if (dma_mapping_error(dev: &adapter->pdev->dev, |
1741 | dma_addr: new_dma_addr)) { |
1742 | put_page(page: new_page); |
1743 | rq->stats.rx_buf_alloc_failure++; |
1744 | dev_kfree_skb(ctx->skb); |
1745 | ctx->skb = NULL; |
1746 | skip_page_frags = true; |
1747 | goto rcd_done; |
1748 | } |
1749 | |
1750 | dma_unmap_page(&adapter->pdev->dev, |
1751 | rbi->dma_addr, rbi->len, |
1752 | DMA_FROM_DEVICE); |
1753 | |
1754 | vmxnet3_append_frag(skb: ctx->skb, rcd, rbi); |
1755 | |
1756 | /* Immediate refill */ |
1757 | rbi->page = new_page; |
1758 | rbi->dma_addr = new_dma_addr; |
1759 | rxd->addr = cpu_to_le64(rbi->dma_addr); |
1760 | rxd->len = rbi->len; |
1761 | } |
1762 | } |
1763 | |
1764 | |
1765 | sop_done: |
1766 | skb = ctx->skb; |
1767 | if (rcd->eop) { |
1768 | u32 mtu = adapter->netdev->mtu; |
1769 | skb->len += skb->data_len; |
1770 | |
1771 | #ifdef VMXNET3_RSS |
1772 | if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE && |
1773 | (adapter->netdev->features & NETIF_F_RXHASH)) { |
1774 | enum pkt_hash_types hash_type; |
1775 | |
1776 | switch (rcd->rssType) { |
1777 | case VMXNET3_RCD_RSS_TYPE_IPV4: |
1778 | case VMXNET3_RCD_RSS_TYPE_IPV6: |
1779 | hash_type = PKT_HASH_TYPE_L3; |
1780 | break; |
1781 | case VMXNET3_RCD_RSS_TYPE_TCPIPV4: |
1782 | case VMXNET3_RCD_RSS_TYPE_TCPIPV6: |
1783 | case VMXNET3_RCD_RSS_TYPE_UDPIPV4: |
1784 | case VMXNET3_RCD_RSS_TYPE_UDPIPV6: |
1785 | hash_type = PKT_HASH_TYPE_L4; |
1786 | break; |
1787 | default: |
1788 | hash_type = PKT_HASH_TYPE_L3; |
1789 | break; |
1790 | } |
1791 | skb_set_hash(skb, |
1792 | le32_to_cpu(rcd->rssHash), |
1793 | type: hash_type); |
1794 | } |
1795 | #endif |
1796 | vmxnet3_rx_csum(adapter, skb, |
1797 | gdesc: (union Vmxnet3_GenericDesc *)rcd); |
1798 | skb->protocol = eth_type_trans(skb, dev: adapter->netdev); |
1799 | if ((!rcd->tcp && !encap_lro) || |
1800 | !(adapter->netdev->features & NETIF_F_LRO)) |
1801 | goto not_lro; |
1802 | |
1803 | if (segCnt != 0 && mss != 0) { |
1804 | skb_shinfo(skb)->gso_type = rcd->v4 ? |
1805 | SKB_GSO_TCPV4 : SKB_GSO_TCPV6; |
1806 | skb_shinfo(skb)->gso_size = mss; |
1807 | skb_shinfo(skb)->gso_segs = segCnt; |
1808 | } else if ((segCnt != 0 || skb->len > mtu) && !encap_lro) { |
1809 | u32 hlen; |
1810 | |
1811 | hlen = vmxnet3_get_hdr_len(adapter, skb, |
1812 | gdesc: (union Vmxnet3_GenericDesc *)rcd); |
1813 | if (hlen == 0) |
1814 | goto not_lro; |
1815 | |
1816 | skb_shinfo(skb)->gso_type = |
1817 | rcd->v4 ? SKB_GSO_TCPV4 : SKB_GSO_TCPV6; |
1818 | if (segCnt != 0) { |
1819 | skb_shinfo(skb)->gso_segs = segCnt; |
1820 | skb_shinfo(skb)->gso_size = |
1821 | DIV_ROUND_UP(skb->len - |
1822 | hlen, segCnt); |
1823 | } else { |
1824 | skb_shinfo(skb)->gso_size = mtu - hlen; |
1825 | } |
1826 | } |
1827 | not_lro: |
1828 | if (unlikely(rcd->ts)) |
1829 | __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tci: rcd->tci); |
1830 | |
1831 | /* Use GRO callback if UPT is enabled */ |
1832 | if ((adapter->netdev->features & NETIF_F_LRO) && |
1833 | !rq->shared->updateRxProd) |
1834 | netif_receive_skb(skb); |
1835 | else |
1836 | napi_gro_receive(napi: &rq->napi, skb); |
1837 | |
1838 | ctx->skb = NULL; |
1839 | encap_lro = false; |
1840 | num_pkts++; |
1841 | } |
1842 | |
1843 | rcd_done: |
1844 | /* device may have skipped some rx descs */ |
1845 | ring = rq->rx_ring + ring_idx; |
1846 | rbi->comp_state = VMXNET3_RXD_COMP_DONE; |
1847 | |
1848 | comp_offset = vmxnet3_cmd_ring_desc_avail(ring); |
1849 | fill_offset = (idx > ring->next2fill ? 0 : ring->size) + |
1850 | idx - ring->next2fill - 1; |
1851 | if (!ring->isOutOfOrder || fill_offset >= comp_offset) |
1852 | ring->next2comp = idx; |
1853 | num_to_alloc = vmxnet3_cmd_ring_desc_avail(ring); |
1854 | |
1855 | /* Ensure that the writes to rxd->gen bits will be observed |
1856 | * after all other writes to rxd objects. |
1857 | */ |
1858 | dma_wmb(); |
1859 | |
1860 | while (num_to_alloc) { |
1861 | rbi = rq->buf_info[ring_idx] + ring->next2fill; |
1862 | if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_OOORX_COMP))) |
1863 | goto refill_buf; |
1864 | if (ring_idx == 0) { |
1865 | /* ring0 Type1 buffers can get skipped; re-fill them */ |
1866 | if (rbi->buf_type != VMXNET3_RX_BUF_SKB) |
1867 | goto refill_buf; |
1868 | } |
1869 | if (rbi->comp_state == VMXNET3_RXD_COMP_DONE) { |
1870 | refill_buf: |
1871 | vmxnet3_getRxDesc(rxd, &ring->base[ring->next2fill].rxd, |
1872 | &rxCmdDesc); |
1873 | WARN_ON(!rxd->addr); |
1874 | |
1875 | /* Recv desc is ready to be used by the device */ |
1876 | rxd->gen = ring->gen; |
1877 | vmxnet3_cmd_ring_adv_next2fill(ring); |
1878 | rbi->comp_state = VMXNET3_RXD_COMP_PENDING; |
1879 | num_to_alloc--; |
1880 | } else { |
1881 | /* rx completion hasn't occurred */ |
1882 | ring->isOutOfOrder = 1; |
1883 | break; |
1884 | } |
1885 | } |
1886 | |
1887 | if (num_to_alloc == 0) { |
1888 | ring->isOutOfOrder = 0; |
1889 | } |
1890 | |
1891 | /* if needed, update the register */ |
1892 | if (unlikely(rq->shared->updateRxProd) && (ring->next2fill & 0xf) == 0) { |
1893 | VMXNET3_WRITE_BAR0_REG(adapter, |
1894 | rxprod_reg[ring_idx] + rq->qid * 8, |
1895 | ring->next2fill); |
1896 | } |
1897 | |
1898 | vmxnet3_comp_ring_adv_next2proc(ring: &rq->comp_ring); |
1899 | vmxnet3_getRxComp(rcd, |
1900 | &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, &rxComp); |
1901 | } |
1902 | if (need_flush) |
1903 | xdp_do_flush(); |
1904 | |
1905 | return num_pkts; |
1906 | } |
1907 | |
1908 | |
1909 | static void |
1910 | vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq, |
1911 | struct vmxnet3_adapter *adapter) |
1912 | { |
1913 | u32 i, ring_idx; |
1914 | struct Vmxnet3_RxDesc *rxd; |
1915 | |
1916 | /* ring has already been cleaned up */ |
1917 | if (!rq->rx_ring[0].base) |
1918 | return; |
1919 | |
1920 | for (ring_idx = 0; ring_idx < 2; ring_idx++) { |
1921 | for (i = 0; i < rq->rx_ring[ring_idx].size; i++) { |
1922 | struct vmxnet3_rx_buf_info *rbi; |
1923 | #ifdef __BIG_ENDIAN_BITFIELD |
1924 | struct Vmxnet3_RxDesc rxDesc; |
1925 | #endif |
1926 | |
1927 | rbi = &rq->buf_info[ring_idx][i]; |
1928 | vmxnet3_getRxDesc(rxd, |
1929 | &rq->rx_ring[ring_idx].base[i].rxd, &rxDesc); |
1930 | |
1931 | if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD && |
1932 | rbi->page && rbi->buf_type == VMXNET3_RX_BUF_XDP) { |
1933 | page_pool_recycle_direct(pool: rq->page_pool, |
1934 | page: rbi->page); |
1935 | rbi->page = NULL; |
1936 | } else if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD && |
1937 | rbi->skb) { |
1938 | dma_unmap_single(&adapter->pdev->dev, rxd->addr, |
1939 | rxd->len, DMA_FROM_DEVICE); |
1940 | dev_kfree_skb(rbi->skb); |
1941 | rbi->skb = NULL; |
1942 | } else if (rxd->btype == VMXNET3_RXD_BTYPE_BODY && |
1943 | rbi->page) { |
1944 | dma_unmap_page(&adapter->pdev->dev, rxd->addr, |
1945 | rxd->len, DMA_FROM_DEVICE); |
1946 | put_page(page: rbi->page); |
1947 | rbi->page = NULL; |
1948 | } |
1949 | } |
1950 | |
1951 | rq->rx_ring[ring_idx].gen = VMXNET3_INIT_GEN; |
1952 | rq->rx_ring[ring_idx].next2fill = |
1953 | rq->rx_ring[ring_idx].next2comp = 0; |
1954 | } |
1955 | |
1956 | rq->comp_ring.gen = VMXNET3_INIT_GEN; |
1957 | rq->comp_ring.next2proc = 0; |
1958 | } |
1959 | |
1960 | |
1961 | static void |
1962 | vmxnet3_rq_cleanup_all(struct vmxnet3_adapter *adapter) |
1963 | { |
1964 | int i; |
1965 | |
1966 | for (i = 0; i < adapter->num_rx_queues; i++) |
1967 | vmxnet3_rq_cleanup(rq: &adapter->rx_queue[i], adapter); |
1968 | rcu_assign_pointer(adapter->xdp_bpf_prog, NULL); |
1969 | } |
1970 | |
1971 | |
1972 | static void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq, |
1973 | struct vmxnet3_adapter *adapter) |
1974 | { |
1975 | int i; |
1976 | int j; |
1977 | |
1978 | /* all rx buffers must have already been freed */ |
1979 | for (i = 0; i < 2; i++) { |
1980 | if (rq->buf_info[i]) { |
1981 | for (j = 0; j < rq->rx_ring[i].size; j++) |
1982 | BUG_ON(rq->buf_info[i][j].page != NULL); |
1983 | } |
1984 | } |
1985 | |
1986 | |
1987 | for (i = 0; i < 2; i++) { |
1988 | if (rq->rx_ring[i].base) { |
1989 | dma_free_coherent(dev: &adapter->pdev->dev, |
1990 | size: rq->rx_ring[i].size |
1991 | * sizeof(struct Vmxnet3_RxDesc), |
1992 | cpu_addr: rq->rx_ring[i].base, |
1993 | dma_handle: rq->rx_ring[i].basePA); |
1994 | rq->rx_ring[i].base = NULL; |
1995 | } |
1996 | } |
1997 | |
1998 | if (xdp_rxq_info_is_reg(xdp_rxq: &rq->xdp_rxq)) |
1999 | xdp_rxq_info_unreg(xdp_rxq: &rq->xdp_rxq); |
2000 | page_pool_destroy(pool: rq->page_pool); |
2001 | rq->page_pool = NULL; |
2002 | |
2003 | if (rq->data_ring.base) { |
2004 | dma_free_coherent(dev: &adapter->pdev->dev, |
2005 | size: rq->rx_ring[0].size * rq->data_ring.desc_size, |
2006 | cpu_addr: rq->data_ring.base, dma_handle: rq->data_ring.basePA); |
2007 | rq->data_ring.base = NULL; |
2008 | } |
2009 | |
2010 | if (rq->comp_ring.base) { |
2011 | dma_free_coherent(dev: &adapter->pdev->dev, size: rq->comp_ring.size |
2012 | * sizeof(struct Vmxnet3_RxCompDesc), |
2013 | cpu_addr: rq->comp_ring.base, dma_handle: rq->comp_ring.basePA); |
2014 | rq->comp_ring.base = NULL; |
2015 | } |
2016 | |
2017 | kfree(objp: rq->buf_info[0]); |
2018 | rq->buf_info[0] = NULL; |
2019 | rq->buf_info[1] = NULL; |
2020 | } |
2021 | |
2022 | static void |
2023 | vmxnet3_rq_destroy_all_rxdataring(struct vmxnet3_adapter *adapter) |
2024 | { |
2025 | int i; |
2026 | |
2027 | for (i = 0; i < adapter->num_rx_queues; i++) { |
2028 | struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; |
2029 | |
2030 | if (rq->data_ring.base) { |
2031 | dma_free_coherent(dev: &adapter->pdev->dev, |
2032 | size: (rq->rx_ring[0].size * |
2033 | rq->data_ring.desc_size), |
2034 | cpu_addr: rq->data_ring.base, |
2035 | dma_handle: rq->data_ring.basePA); |
2036 | rq->data_ring.base = NULL; |
2037 | rq->data_ring.desc_size = 0; |
2038 | } |
2039 | } |
2040 | } |
2041 | |
2042 | static int |
2043 | vmxnet3_rq_init(struct vmxnet3_rx_queue *rq, |
2044 | struct vmxnet3_adapter *adapter) |
2045 | { |
2046 | int i, err; |
2047 | |
2048 | /* initialize buf_info */ |
2049 | for (i = 0; i < rq->rx_ring[0].size; i++) { |
2050 | |
2051 | /* 1st buf for a pkt is skbuff or xdp page */ |
2052 | if (i % adapter->rx_buf_per_pkt == 0) { |
2053 | rq->buf_info[0][i].buf_type = vmxnet3_xdp_enabled(adapter) ? |
2054 | VMXNET3_RX_BUF_XDP : |
2055 | VMXNET3_RX_BUF_SKB; |
2056 | rq->buf_info[0][i].len = adapter->skb_buf_size; |
2057 | } else { /* subsequent bufs for a pkt is frag */ |
2058 | rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_PAGE; |
2059 | rq->buf_info[0][i].len = PAGE_SIZE; |
2060 | } |
2061 | } |
2062 | for (i = 0; i < rq->rx_ring[1].size; i++) { |
2063 | rq->buf_info[1][i].buf_type = VMXNET3_RX_BUF_PAGE; |
2064 | rq->buf_info[1][i].len = PAGE_SIZE; |
2065 | } |
2066 | |
2067 | /* reset internal state and allocate buffers for both rings */ |
2068 | for (i = 0; i < 2; i++) { |
2069 | rq->rx_ring[i].next2fill = rq->rx_ring[i].next2comp = 0; |
2070 | |
2071 | memset(rq->rx_ring[i].base, 0, rq->rx_ring[i].size * |
2072 | sizeof(struct Vmxnet3_RxDesc)); |
2073 | rq->rx_ring[i].gen = VMXNET3_INIT_GEN; |
2074 | rq->rx_ring[i].isOutOfOrder = 0; |
2075 | } |
2076 | |
2077 | err = vmxnet3_create_pp(adapter, rq, |
2078 | size: rq->rx_ring[0].size + rq->rx_ring[1].size); |
2079 | if (err) |
2080 | return err; |
2081 | |
2082 | if (vmxnet3_rq_alloc_rx_buf(rq, ring_idx: 0, num_to_alloc: rq->rx_ring[0].size - 1, |
2083 | adapter) == 0) { |
2084 | xdp_rxq_info_unreg(xdp_rxq: &rq->xdp_rxq); |
2085 | page_pool_destroy(pool: rq->page_pool); |
2086 | rq->page_pool = NULL; |
2087 | |
2088 | /* at least has 1 rx buffer for the 1st ring */ |
2089 | return -ENOMEM; |
2090 | } |
2091 | vmxnet3_rq_alloc_rx_buf(rq, ring_idx: 1, num_to_alloc: rq->rx_ring[1].size - 1, adapter); |
2092 | |
2093 | /* reset the comp ring */ |
2094 | rq->comp_ring.next2proc = 0; |
2095 | memset(rq->comp_ring.base, 0, rq->comp_ring.size * |
2096 | sizeof(struct Vmxnet3_RxCompDesc)); |
2097 | rq->comp_ring.gen = VMXNET3_INIT_GEN; |
2098 | |
2099 | /* reset rxctx */ |
2100 | rq->rx_ctx.skb = NULL; |
2101 | |
2102 | /* stats are not reset */ |
2103 | return 0; |
2104 | } |
2105 | |
2106 | |
2107 | static int |
2108 | vmxnet3_rq_init_all(struct vmxnet3_adapter *adapter) |
2109 | { |
2110 | int i, err = 0; |
2111 | |
2112 | for (i = 0; i < adapter->num_rx_queues; i++) { |
2113 | err = vmxnet3_rq_init(rq: &adapter->rx_queue[i], adapter); |
2114 | if (unlikely(err)) { |
2115 | dev_err(&adapter->netdev->dev, "%s: failed to " |
2116 | "initialize rx queue%i\n" , |
2117 | adapter->netdev->name, i); |
2118 | break; |
2119 | } |
2120 | } |
2121 | return err; |
2122 | |
2123 | } |
2124 | |
2125 | |
2126 | static int |
2127 | vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter) |
2128 | { |
2129 | int i; |
2130 | size_t sz; |
2131 | struct vmxnet3_rx_buf_info *bi; |
2132 | |
2133 | for (i = 0; i < 2; i++) { |
2134 | |
2135 | sz = rq->rx_ring[i].size * sizeof(struct Vmxnet3_RxDesc); |
2136 | rq->rx_ring[i].base = dma_alloc_coherent( |
2137 | dev: &adapter->pdev->dev, size: sz, |
2138 | dma_handle: &rq->rx_ring[i].basePA, |
2139 | GFP_KERNEL); |
2140 | if (!rq->rx_ring[i].base) { |
2141 | netdev_err(dev: adapter->netdev, |
2142 | format: "failed to allocate rx ring %d\n" , i); |
2143 | goto err; |
2144 | } |
2145 | } |
2146 | |
2147 | if ((adapter->rxdataring_enabled) && (rq->data_ring.desc_size != 0)) { |
2148 | sz = rq->rx_ring[0].size * rq->data_ring.desc_size; |
2149 | rq->data_ring.base = |
2150 | dma_alloc_coherent(dev: &adapter->pdev->dev, size: sz, |
2151 | dma_handle: &rq->data_ring.basePA, |
2152 | GFP_KERNEL); |
2153 | if (!rq->data_ring.base) { |
2154 | netdev_err(dev: adapter->netdev, |
2155 | format: "rx data ring will be disabled\n" ); |
2156 | adapter->rxdataring_enabled = false; |
2157 | } |
2158 | } else { |
2159 | rq->data_ring.base = NULL; |
2160 | rq->data_ring.desc_size = 0; |
2161 | } |
2162 | |
2163 | sz = rq->comp_ring.size * sizeof(struct Vmxnet3_RxCompDesc); |
2164 | rq->comp_ring.base = dma_alloc_coherent(dev: &adapter->pdev->dev, size: sz, |
2165 | dma_handle: &rq->comp_ring.basePA, |
2166 | GFP_KERNEL); |
2167 | if (!rq->comp_ring.base) { |
2168 | netdev_err(dev: adapter->netdev, format: "failed to allocate rx comp ring\n" ); |
2169 | goto err; |
2170 | } |
2171 | |
2172 | bi = kcalloc_node(n: rq->rx_ring[0].size + rq->rx_ring[1].size, |
2173 | size: sizeof(rq->buf_info[0][0]), GFP_KERNEL, |
2174 | node: dev_to_node(dev: &adapter->pdev->dev)); |
2175 | if (!bi) |
2176 | goto err; |
2177 | |
2178 | rq->buf_info[0] = bi; |
2179 | rq->buf_info[1] = bi + rq->rx_ring[0].size; |
2180 | |
2181 | return 0; |
2182 | |
2183 | err: |
2184 | vmxnet3_rq_destroy(rq, adapter); |
2185 | return -ENOMEM; |
2186 | } |
2187 | |
2188 | |
2189 | int |
2190 | vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter) |
2191 | { |
2192 | int i, err = 0; |
2193 | |
2194 | adapter->rxdataring_enabled = VMXNET3_VERSION_GE_3(adapter); |
2195 | |
2196 | for (i = 0; i < adapter->num_rx_queues; i++) { |
2197 | err = vmxnet3_rq_create(rq: &adapter->rx_queue[i], adapter); |
2198 | if (unlikely(err)) { |
2199 | dev_err(&adapter->netdev->dev, |
2200 | "%s: failed to create rx queue%i\n" , |
2201 | adapter->netdev->name, i); |
2202 | goto err_out; |
2203 | } |
2204 | } |
2205 | |
2206 | if (!adapter->rxdataring_enabled) |
2207 | vmxnet3_rq_destroy_all_rxdataring(adapter); |
2208 | |
2209 | return err; |
2210 | err_out: |
2211 | vmxnet3_rq_destroy_all(adapter); |
2212 | return err; |
2213 | |
2214 | } |
2215 | |
2216 | /* Multiple queue aware polling function for tx and rx */ |
2217 | |
2218 | static int |
2219 | vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget) |
2220 | { |
2221 | int rcd_done = 0, i; |
2222 | if (unlikely(adapter->shared->ecr)) |
2223 | vmxnet3_process_events(adapter); |
2224 | for (i = 0; i < adapter->num_tx_queues; i++) |
2225 | vmxnet3_tq_tx_complete(tq: &adapter->tx_queue[i], adapter); |
2226 | |
2227 | for (i = 0; i < adapter->num_rx_queues; i++) |
2228 | rcd_done += vmxnet3_rq_rx_complete(rq: &adapter->rx_queue[i], |
2229 | adapter, quota: budget); |
2230 | return rcd_done; |
2231 | } |
2232 | |
2233 | |
2234 | static int |
2235 | vmxnet3_poll(struct napi_struct *napi, int budget) |
2236 | { |
2237 | struct vmxnet3_rx_queue *rx_queue = container_of(napi, |
2238 | struct vmxnet3_rx_queue, napi); |
2239 | int rxd_done; |
2240 | |
2241 | rxd_done = vmxnet3_do_poll(adapter: rx_queue->adapter, budget); |
2242 | |
2243 | if (rxd_done < budget) { |
2244 | napi_complete_done(n: napi, work_done: rxd_done); |
2245 | vmxnet3_enable_all_intrs(adapter: rx_queue->adapter); |
2246 | } |
2247 | return rxd_done; |
2248 | } |
2249 | |
2250 | /* |
2251 | * NAPI polling function for MSI-X mode with multiple Rx queues |
2252 | * Returns the # of the NAPI credit consumed (# of rx descriptors processed) |
2253 | */ |
2254 | |
2255 | static int |
2256 | vmxnet3_poll_rx_only(struct napi_struct *napi, int budget) |
2257 | { |
2258 | struct vmxnet3_rx_queue *rq = container_of(napi, |
2259 | struct vmxnet3_rx_queue, napi); |
2260 | struct vmxnet3_adapter *adapter = rq->adapter; |
2261 | int rxd_done; |
2262 | |
2263 | /* When sharing interrupt with corresponding tx queue, process |
2264 | * tx completions in that queue as well |
2265 | */ |
2266 | if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) { |
2267 | struct vmxnet3_tx_queue *tq = |
2268 | &adapter->tx_queue[rq - adapter->rx_queue]; |
2269 | vmxnet3_tq_tx_complete(tq, adapter); |
2270 | } |
2271 | |
2272 | rxd_done = vmxnet3_rq_rx_complete(rq, adapter, quota: budget); |
2273 | |
2274 | if (rxd_done < budget) { |
2275 | napi_complete_done(n: napi, work_done: rxd_done); |
2276 | vmxnet3_enable_intr(adapter, intr_idx: rq->comp_ring.intr_idx); |
2277 | } |
2278 | return rxd_done; |
2279 | } |
2280 | |
2281 | |
2282 | #ifdef CONFIG_PCI_MSI |
2283 | |
2284 | /* |
2285 | * Handle completion interrupts on tx queues |
2286 | * Returns whether or not the intr is handled |
2287 | */ |
2288 | |
2289 | static irqreturn_t |
2290 | vmxnet3_msix_tx(int irq, void *data) |
2291 | { |
2292 | struct vmxnet3_tx_queue *tq = data; |
2293 | struct vmxnet3_adapter *adapter = tq->adapter; |
2294 | |
2295 | if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) |
2296 | vmxnet3_disable_intr(adapter, intr_idx: tq->comp_ring.intr_idx); |
2297 | |
2298 | /* Handle the case where only one irq is allocate for all tx queues */ |
2299 | if (adapter->share_intr == VMXNET3_INTR_TXSHARE) { |
2300 | int i; |
2301 | for (i = 0; i < adapter->num_tx_queues; i++) { |
2302 | struct vmxnet3_tx_queue *txq = &adapter->tx_queue[i]; |
2303 | vmxnet3_tq_tx_complete(tq: txq, adapter); |
2304 | } |
2305 | } else { |
2306 | vmxnet3_tq_tx_complete(tq, adapter); |
2307 | } |
2308 | vmxnet3_enable_intr(adapter, intr_idx: tq->comp_ring.intr_idx); |
2309 | |
2310 | return IRQ_HANDLED; |
2311 | } |
2312 | |
2313 | |
2314 | /* |
2315 | * Handle completion interrupts on rx queues. Returns whether or not the |
2316 | * intr is handled |
2317 | */ |
2318 | |
2319 | static irqreturn_t |
2320 | vmxnet3_msix_rx(int irq, void *data) |
2321 | { |
2322 | struct vmxnet3_rx_queue *rq = data; |
2323 | struct vmxnet3_adapter *adapter = rq->adapter; |
2324 | |
2325 | /* disable intr if needed */ |
2326 | if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) |
2327 | vmxnet3_disable_intr(adapter, intr_idx: rq->comp_ring.intr_idx); |
2328 | napi_schedule(n: &rq->napi); |
2329 | |
2330 | return IRQ_HANDLED; |
2331 | } |
2332 | |
2333 | /* |
2334 | *---------------------------------------------------------------------------- |
2335 | * |
2336 | * vmxnet3_msix_event -- |
2337 | * |
2338 | * vmxnet3 msix event intr handler |
2339 | * |
2340 | * Result: |
2341 | * whether or not the intr is handled |
2342 | * |
2343 | *---------------------------------------------------------------------------- |
2344 | */ |
2345 | |
2346 | static irqreturn_t |
2347 | vmxnet3_msix_event(int irq, void *data) |
2348 | { |
2349 | struct net_device *dev = data; |
2350 | struct vmxnet3_adapter *adapter = netdev_priv(dev); |
2351 | |
2352 | /* disable intr if needed */ |
2353 | if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) |
2354 | vmxnet3_disable_intr(adapter, intr_idx: adapter->intr.event_intr_idx); |
2355 | |
2356 | if (adapter->shared->ecr) |
2357 | vmxnet3_process_events(adapter); |
2358 | |
2359 | vmxnet3_enable_intr(adapter, intr_idx: adapter->intr.event_intr_idx); |
2360 | |
2361 | return IRQ_HANDLED; |
2362 | } |
2363 | |
2364 | #endif /* CONFIG_PCI_MSI */ |
2365 | |
2366 | |
2367 | /* Interrupt handler for vmxnet3 */ |
2368 | static irqreturn_t |
2369 | vmxnet3_intr(int irq, void *dev_id) |
2370 | { |
2371 | struct net_device *dev = dev_id; |
2372 | struct vmxnet3_adapter *adapter = netdev_priv(dev); |
2373 | |
2374 | if (adapter->intr.type == VMXNET3_IT_INTX) { |
2375 | u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR); |
2376 | if (unlikely(icr == 0)) |
2377 | /* not ours */ |
2378 | return IRQ_NONE; |
2379 | } |
2380 | |
2381 | |
2382 | /* disable intr if needed */ |
2383 | if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) |
2384 | vmxnet3_disable_all_intrs(adapter); |
2385 | |
2386 | napi_schedule(n: &adapter->rx_queue[0].napi); |
2387 | |
2388 | return IRQ_HANDLED; |
2389 | } |
2390 | |
2391 | #ifdef CONFIG_NET_POLL_CONTROLLER |
2392 | |
2393 | /* netpoll callback. */ |
2394 | static void |
2395 | vmxnet3_netpoll(struct net_device *netdev) |
2396 | { |
2397 | struct vmxnet3_adapter *adapter = netdev_priv(dev: netdev); |
2398 | |
2399 | switch (adapter->intr.type) { |
2400 | #ifdef CONFIG_PCI_MSI |
2401 | case VMXNET3_IT_MSIX: { |
2402 | int i; |
2403 | for (i = 0; i < adapter->num_rx_queues; i++) |
2404 | vmxnet3_msix_rx(irq: 0, data: &adapter->rx_queue[i]); |
2405 | break; |
2406 | } |
2407 | #endif |
2408 | case VMXNET3_IT_MSI: |
2409 | default: |
2410 | vmxnet3_intr(irq: 0, dev_id: adapter->netdev); |
2411 | break; |
2412 | } |
2413 | |
2414 | } |
2415 | #endif /* CONFIG_NET_POLL_CONTROLLER */ |
2416 | |
2417 | static int |
2418 | vmxnet3_request_irqs(struct vmxnet3_adapter *adapter) |
2419 | { |
2420 | struct vmxnet3_intr *intr = &adapter->intr; |
2421 | int err = 0, i; |
2422 | int vector = 0; |
2423 | |
2424 | #ifdef CONFIG_PCI_MSI |
2425 | if (adapter->intr.type == VMXNET3_IT_MSIX) { |
2426 | for (i = 0; i < adapter->num_tx_queues; i++) { |
2427 | if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) { |
2428 | sprintf(buf: adapter->tx_queue[i].name, fmt: "%s-tx-%d" , |
2429 | adapter->netdev->name, vector); |
2430 | err = request_irq( |
2431 | irq: intr->msix_entries[vector].vector, |
2432 | handler: vmxnet3_msix_tx, flags: 0, |
2433 | name: adapter->tx_queue[i].name, |
2434 | dev: &adapter->tx_queue[i]); |
2435 | } else { |
2436 | sprintf(buf: adapter->tx_queue[i].name, fmt: "%s-rxtx-%d" , |
2437 | adapter->netdev->name, vector); |
2438 | } |
2439 | if (err) { |
2440 | dev_err(&adapter->netdev->dev, |
2441 | "Failed to request irq for MSIX, %s, " |
2442 | "error %d\n" , |
2443 | adapter->tx_queue[i].name, err); |
2444 | return err; |
2445 | } |
2446 | |
2447 | /* Handle the case where only 1 MSIx was allocated for |
2448 | * all tx queues */ |
2449 | if (adapter->share_intr == VMXNET3_INTR_TXSHARE) { |
2450 | for (; i < adapter->num_tx_queues; i++) |
2451 | adapter->tx_queue[i].comp_ring.intr_idx |
2452 | = vector; |
2453 | vector++; |
2454 | break; |
2455 | } else { |
2456 | adapter->tx_queue[i].comp_ring.intr_idx |
2457 | = vector++; |
2458 | } |
2459 | } |
2460 | if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) |
2461 | vector = 0; |
2462 | |
2463 | for (i = 0; i < adapter->num_rx_queues; i++) { |
2464 | if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) |
2465 | sprintf(buf: adapter->rx_queue[i].name, fmt: "%s-rx-%d" , |
2466 | adapter->netdev->name, vector); |
2467 | else |
2468 | sprintf(buf: adapter->rx_queue[i].name, fmt: "%s-rxtx-%d" , |
2469 | adapter->netdev->name, vector); |
2470 | err = request_irq(irq: intr->msix_entries[vector].vector, |
2471 | handler: vmxnet3_msix_rx, flags: 0, |
2472 | name: adapter->rx_queue[i].name, |
2473 | dev: &(adapter->rx_queue[i])); |
2474 | if (err) { |
2475 | netdev_err(dev: adapter->netdev, |
2476 | format: "Failed to request irq for MSIX, " |
2477 | "%s, error %d\n" , |
2478 | adapter->rx_queue[i].name, err); |
2479 | return err; |
2480 | } |
2481 | |
2482 | adapter->rx_queue[i].comp_ring.intr_idx = vector++; |
2483 | } |
2484 | |
2485 | sprintf(buf: intr->event_msi_vector_name, fmt: "%s-event-%d" , |
2486 | adapter->netdev->name, vector); |
2487 | err = request_irq(irq: intr->msix_entries[vector].vector, |
2488 | handler: vmxnet3_msix_event, flags: 0, |
2489 | name: intr->event_msi_vector_name, dev: adapter->netdev); |
2490 | intr->event_intr_idx = vector; |
2491 | |
2492 | } else if (intr->type == VMXNET3_IT_MSI) { |
2493 | adapter->num_rx_queues = 1; |
2494 | err = request_irq(irq: adapter->pdev->irq, handler: vmxnet3_intr, flags: 0, |
2495 | name: adapter->netdev->name, dev: adapter->netdev); |
2496 | } else { |
2497 | #endif |
2498 | adapter->num_rx_queues = 1; |
2499 | err = request_irq(irq: adapter->pdev->irq, handler: vmxnet3_intr, |
2500 | IRQF_SHARED, name: adapter->netdev->name, |
2501 | dev: adapter->netdev); |
2502 | #ifdef CONFIG_PCI_MSI |
2503 | } |
2504 | #endif |
2505 | intr->num_intrs = vector + 1; |
2506 | if (err) { |
2507 | netdev_err(dev: adapter->netdev, |
2508 | format: "Failed to request irq (intr type:%d), error %d\n" , |
2509 | intr->type, err); |
2510 | } else { |
2511 | /* Number of rx queues will not change after this */ |
2512 | for (i = 0; i < adapter->num_rx_queues; i++) { |
2513 | struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; |
2514 | rq->qid = i; |
2515 | rq->qid2 = i + adapter->num_rx_queues; |
2516 | rq->dataRingQid = i + 2 * adapter->num_rx_queues; |
2517 | } |
2518 | |
2519 | /* init our intr settings */ |
2520 | for (i = 0; i < intr->num_intrs; i++) |
2521 | intr->mod_levels[i] = UPT1_IML_ADAPTIVE; |
2522 | if (adapter->intr.type != VMXNET3_IT_MSIX) { |
2523 | adapter->intr.event_intr_idx = 0; |
2524 | for (i = 0; i < adapter->num_tx_queues; i++) |
2525 | adapter->tx_queue[i].comp_ring.intr_idx = 0; |
2526 | adapter->rx_queue[0].comp_ring.intr_idx = 0; |
2527 | } |
2528 | |
2529 | netdev_info(dev: adapter->netdev, |
2530 | format: "intr type %u, mode %u, %u vectors allocated\n" , |
2531 | intr->type, intr->mask_mode, intr->num_intrs); |
2532 | } |
2533 | |
2534 | return err; |
2535 | } |
2536 | |
2537 | |
2538 | static void |
2539 | vmxnet3_free_irqs(struct vmxnet3_adapter *adapter) |
2540 | { |
2541 | struct vmxnet3_intr *intr = &adapter->intr; |
2542 | BUG_ON(intr->type == VMXNET3_IT_AUTO || intr->num_intrs <= 0); |
2543 | |
2544 | switch (intr->type) { |
2545 | #ifdef CONFIG_PCI_MSI |
2546 | case VMXNET3_IT_MSIX: |
2547 | { |
2548 | int i, vector = 0; |
2549 | |
2550 | if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) { |
2551 | for (i = 0; i < adapter->num_tx_queues; i++) { |
2552 | free_irq(intr->msix_entries[vector++].vector, |
2553 | &(adapter->tx_queue[i])); |
2554 | if (adapter->share_intr == VMXNET3_INTR_TXSHARE) |
2555 | break; |
2556 | } |
2557 | } |
2558 | |
2559 | for (i = 0; i < adapter->num_rx_queues; i++) { |
2560 | free_irq(intr->msix_entries[vector++].vector, |
2561 | &(adapter->rx_queue[i])); |
2562 | } |
2563 | |
2564 | free_irq(intr->msix_entries[vector].vector, |
2565 | adapter->netdev); |
2566 | BUG_ON(vector >= intr->num_intrs); |
2567 | break; |
2568 | } |
2569 | #endif |
2570 | case VMXNET3_IT_MSI: |
2571 | free_irq(adapter->pdev->irq, adapter->netdev); |
2572 | break; |
2573 | case VMXNET3_IT_INTX: |
2574 | free_irq(adapter->pdev->irq, adapter->netdev); |
2575 | break; |
2576 | default: |
2577 | BUG(); |
2578 | } |
2579 | } |
2580 | |
2581 | |
2582 | static void |
2583 | vmxnet3_restore_vlan(struct vmxnet3_adapter *adapter) |
2584 | { |
2585 | u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; |
2586 | u16 vid; |
2587 | |
2588 | /* allow untagged pkts */ |
2589 | VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0); |
2590 | |
2591 | for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) |
2592 | VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid); |
2593 | } |
2594 | |
2595 | |
2596 | static int |
2597 | vmxnet3_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid) |
2598 | { |
2599 | struct vmxnet3_adapter *adapter = netdev_priv(dev: netdev); |
2600 | |
2601 | if (!(netdev->flags & IFF_PROMISC)) { |
2602 | u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; |
2603 | unsigned long flags; |
2604 | |
2605 | VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid); |
2606 | spin_lock_irqsave(&adapter->cmd_lock, flags); |
2607 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, |
2608 | VMXNET3_CMD_UPDATE_VLAN_FILTERS); |
2609 | spin_unlock_irqrestore(lock: &adapter->cmd_lock, flags); |
2610 | } |
2611 | |
2612 | set_bit(nr: vid, addr: adapter->active_vlans); |
2613 | |
2614 | return 0; |
2615 | } |
2616 | |
2617 | |
2618 | static int |
2619 | vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid) |
2620 | { |
2621 | struct vmxnet3_adapter *adapter = netdev_priv(dev: netdev); |
2622 | |
2623 | if (!(netdev->flags & IFF_PROMISC)) { |
2624 | u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; |
2625 | unsigned long flags; |
2626 | |
2627 | VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid); |
2628 | spin_lock_irqsave(&adapter->cmd_lock, flags); |
2629 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, |
2630 | VMXNET3_CMD_UPDATE_VLAN_FILTERS); |
2631 | spin_unlock_irqrestore(lock: &adapter->cmd_lock, flags); |
2632 | } |
2633 | |
2634 | clear_bit(nr: vid, addr: adapter->active_vlans); |
2635 | |
2636 | return 0; |
2637 | } |
2638 | |
2639 | |
2640 | static u8 * |
2641 | vmxnet3_copy_mc(struct net_device *netdev) |
2642 | { |
2643 | u8 *buf = NULL; |
2644 | u32 sz = netdev_mc_count(netdev) * ETH_ALEN; |
2645 | |
2646 | /* struct Vmxnet3_RxFilterConf.mfTableLen is u16. */ |
2647 | if (sz <= 0xffff) { |
2648 | /* We may be called with BH disabled */ |
2649 | buf = kmalloc(size: sz, GFP_ATOMIC); |
2650 | if (buf) { |
2651 | struct netdev_hw_addr *ha; |
2652 | int i = 0; |
2653 | |
2654 | netdev_for_each_mc_addr(ha, netdev) |
2655 | memcpy(buf + i++ * ETH_ALEN, ha->addr, |
2656 | ETH_ALEN); |
2657 | } |
2658 | } |
2659 | return buf; |
2660 | } |
2661 | |
2662 | |
2663 | static void |
2664 | vmxnet3_set_mc(struct net_device *netdev) |
2665 | { |
2666 | struct vmxnet3_adapter *adapter = netdev_priv(dev: netdev); |
2667 | unsigned long flags; |
2668 | struct Vmxnet3_RxFilterConf *rxConf = |
2669 | &adapter->shared->devRead.rxFilterConf; |
2670 | u8 *new_table = NULL; |
2671 | dma_addr_t new_table_pa = 0; |
2672 | bool new_table_pa_valid = false; |
2673 | u32 new_mode = VMXNET3_RXM_UCAST; |
2674 | |
2675 | if (netdev->flags & IFF_PROMISC) { |
2676 | u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; |
2677 | memset(vfTable, 0, VMXNET3_VFT_SIZE * sizeof(*vfTable)); |
2678 | |
2679 | new_mode |= VMXNET3_RXM_PROMISC; |
2680 | } else { |
2681 | vmxnet3_restore_vlan(adapter); |
2682 | } |
2683 | |
2684 | if (netdev->flags & IFF_BROADCAST) |
2685 | new_mode |= VMXNET3_RXM_BCAST; |
2686 | |
2687 | if (netdev->flags & IFF_ALLMULTI) |
2688 | new_mode |= VMXNET3_RXM_ALL_MULTI; |
2689 | else |
2690 | if (!netdev_mc_empty(netdev)) { |
2691 | new_table = vmxnet3_copy_mc(netdev); |
2692 | if (new_table) { |
2693 | size_t sz = netdev_mc_count(netdev) * ETH_ALEN; |
2694 | |
2695 | rxConf->mfTableLen = cpu_to_le16(sz); |
2696 | new_table_pa = dma_map_single( |
2697 | &adapter->pdev->dev, |
2698 | new_table, |
2699 | sz, |
2700 | DMA_TO_DEVICE); |
2701 | if (!dma_mapping_error(dev: &adapter->pdev->dev, |
2702 | dma_addr: new_table_pa)) { |
2703 | new_mode |= VMXNET3_RXM_MCAST; |
2704 | new_table_pa_valid = true; |
2705 | rxConf->mfTablePA = cpu_to_le64( |
2706 | new_table_pa); |
2707 | } |
2708 | } |
2709 | if (!new_table_pa_valid) { |
2710 | netdev_info(dev: netdev, |
2711 | format: "failed to copy mcast list, setting ALL_MULTI\n" ); |
2712 | new_mode |= VMXNET3_RXM_ALL_MULTI; |
2713 | } |
2714 | } |
2715 | |
2716 | if (!(new_mode & VMXNET3_RXM_MCAST)) { |
2717 | rxConf->mfTableLen = 0; |
2718 | rxConf->mfTablePA = 0; |
2719 | } |
2720 | |
2721 | spin_lock_irqsave(&adapter->cmd_lock, flags); |
2722 | if (new_mode != rxConf->rxMode) { |
2723 | rxConf->rxMode = cpu_to_le32(new_mode); |
2724 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, |
2725 | VMXNET3_CMD_UPDATE_RX_MODE); |
2726 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, |
2727 | VMXNET3_CMD_UPDATE_VLAN_FILTERS); |
2728 | } |
2729 | |
2730 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, |
2731 | VMXNET3_CMD_UPDATE_MAC_FILTERS); |
2732 | spin_unlock_irqrestore(lock: &adapter->cmd_lock, flags); |
2733 | |
2734 | if (new_table_pa_valid) |
2735 | dma_unmap_single(&adapter->pdev->dev, new_table_pa, |
2736 | rxConf->mfTableLen, DMA_TO_DEVICE); |
2737 | kfree(objp: new_table); |
2738 | } |
2739 | |
2740 | void |
2741 | vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter) |
2742 | { |
2743 | int i; |
2744 | |
2745 | for (i = 0; i < adapter->num_rx_queues; i++) |
2746 | vmxnet3_rq_destroy(rq: &adapter->rx_queue[i], adapter); |
2747 | } |
2748 | |
2749 | |
2750 | /* |
2751 | * Set up driver_shared based on settings in adapter. |
2752 | */ |
2753 | |
2754 | static void |
2755 | vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter) |
2756 | { |
2757 | struct Vmxnet3_DriverShared *shared = adapter->shared; |
2758 | struct Vmxnet3_DSDevRead *devRead = &shared->devRead; |
2759 | struct Vmxnet3_DSDevReadExt *devReadExt = &shared->devReadExt; |
2760 | struct Vmxnet3_TxQueueConf *tqc; |
2761 | struct Vmxnet3_RxQueueConf *rqc; |
2762 | int i; |
2763 | |
2764 | memset(shared, 0, sizeof(*shared)); |
2765 | |
2766 | /* driver settings */ |
2767 | shared->magic = cpu_to_le32(VMXNET3_REV1_MAGIC); |
2768 | devRead->misc.driverInfo.version = cpu_to_le32( |
2769 | VMXNET3_DRIVER_VERSION_NUM); |
2770 | devRead->misc.driverInfo.gos.gosBits = (sizeof(void *) == 4 ? |
2771 | VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64); |
2772 | devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX; |
2773 | *((u32 *)&devRead->misc.driverInfo.gos) = cpu_to_le32( |
2774 | *((u32 *)&devRead->misc.driverInfo.gos)); |
2775 | devRead->misc.driverInfo.vmxnet3RevSpt = cpu_to_le32(1); |
2776 | devRead->misc.driverInfo.uptVerSpt = cpu_to_le32(1); |
2777 | |
2778 | devRead->misc.ddPA = cpu_to_le64(adapter->adapter_pa); |
2779 | devRead->misc.ddLen = cpu_to_le32(sizeof(struct vmxnet3_adapter)); |
2780 | |
2781 | /* set up feature flags */ |
2782 | if (adapter->netdev->features & NETIF_F_RXCSUM) |
2783 | devRead->misc.uptFeatures |= UPT1_F_RXCSUM; |
2784 | |
2785 | if (adapter->netdev->features & NETIF_F_LRO) { |
2786 | devRead->misc.uptFeatures |= UPT1_F_LRO; |
2787 | devRead->misc.maxNumRxSG = cpu_to_le16(1 + MAX_SKB_FRAGS); |
2788 | } |
2789 | if (adapter->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) |
2790 | devRead->misc.uptFeatures |= UPT1_F_RXVLAN; |
2791 | |
2792 | if (adapter->netdev->features & (NETIF_F_GSO_UDP_TUNNEL | |
2793 | NETIF_F_GSO_UDP_TUNNEL_CSUM)) |
2794 | devRead->misc.uptFeatures |= UPT1_F_RXINNEROFLD; |
2795 | |
2796 | devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu); |
2797 | devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa); |
2798 | devRead->misc.queueDescLen = cpu_to_le32( |
2799 | adapter->num_tx_queues * sizeof(struct Vmxnet3_TxQueueDesc) + |
2800 | adapter->num_rx_queues * sizeof(struct Vmxnet3_RxQueueDesc)); |
2801 | |
2802 | /* tx queue settings */ |
2803 | devRead->misc.numTxQueues = adapter->num_tx_queues; |
2804 | for (i = 0; i < adapter->num_tx_queues; i++) { |
2805 | struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i]; |
2806 | BUG_ON(adapter->tx_queue[i].tx_ring.base == NULL); |
2807 | tqc = &adapter->tqd_start[i].conf; |
2808 | tqc->txRingBasePA = cpu_to_le64(tq->tx_ring.basePA); |
2809 | tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA); |
2810 | tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA); |
2811 | tqc->ddPA = cpu_to_le64(~0ULL); |
2812 | tqc->txRingSize = cpu_to_le32(tq->tx_ring.size); |
2813 | tqc->dataRingSize = cpu_to_le32(tq->data_ring.size); |
2814 | tqc->txDataRingDescSize = cpu_to_le32(tq->txdata_desc_size); |
2815 | tqc->compRingSize = cpu_to_le32(tq->comp_ring.size); |
2816 | tqc->ddLen = cpu_to_le32(0); |
2817 | tqc->intrIdx = tq->comp_ring.intr_idx; |
2818 | } |
2819 | |
2820 | /* rx queue settings */ |
2821 | devRead->misc.numRxQueues = adapter->num_rx_queues; |
2822 | for (i = 0; i < adapter->num_rx_queues; i++) { |
2823 | struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; |
2824 | rqc = &adapter->rqd_start[i].conf; |
2825 | rqc->rxRingBasePA[0] = cpu_to_le64(rq->rx_ring[0].basePA); |
2826 | rqc->rxRingBasePA[1] = cpu_to_le64(rq->rx_ring[1].basePA); |
2827 | rqc->compRingBasePA = cpu_to_le64(rq->comp_ring.basePA); |
2828 | rqc->ddPA = cpu_to_le64(~0ULL); |
2829 | rqc->rxRingSize[0] = cpu_to_le32(rq->rx_ring[0].size); |
2830 | rqc->rxRingSize[1] = cpu_to_le32(rq->rx_ring[1].size); |
2831 | rqc->compRingSize = cpu_to_le32(rq->comp_ring.size); |
2832 | rqc->ddLen = cpu_to_le32(0); |
2833 | rqc->intrIdx = rq->comp_ring.intr_idx; |
2834 | if (VMXNET3_VERSION_GE_3(adapter)) { |
2835 | rqc->rxDataRingBasePA = |
2836 | cpu_to_le64(rq->data_ring.basePA); |
2837 | rqc->rxDataRingDescSize = |
2838 | cpu_to_le16(rq->data_ring.desc_size); |
2839 | } |
2840 | } |
2841 | |
2842 | #ifdef VMXNET3_RSS |
2843 | memset(adapter->rss_conf, 0, sizeof(*adapter->rss_conf)); |
2844 | |
2845 | if (adapter->rss) { |
2846 | struct UPT1_RSSConf * = adapter->rss_conf; |
2847 | |
2848 | devRead->misc.uptFeatures |= UPT1_F_RSS; |
2849 | devRead->misc.numRxQueues = adapter->num_rx_queues; |
2850 | rssConf->hashType = UPT1_RSS_HASH_TYPE_TCP_IPV4 | |
2851 | UPT1_RSS_HASH_TYPE_IPV4 | |
2852 | UPT1_RSS_HASH_TYPE_TCP_IPV6 | |
2853 | UPT1_RSS_HASH_TYPE_IPV6; |
2854 | rssConf->hashFunc = UPT1_RSS_HASH_FUNC_TOEPLITZ; |
2855 | rssConf->hashKeySize = UPT1_RSS_MAX_KEY_SIZE; |
2856 | rssConf->indTableSize = VMXNET3_RSS_IND_TABLE_SIZE; |
2857 | netdev_rss_key_fill(buffer: rssConf->hashKey, len: sizeof(rssConf->hashKey)); |
2858 | |
2859 | for (i = 0; i < rssConf->indTableSize; i++) |
2860 | rssConf->indTable[i] = ethtool_rxfh_indir_default( |
2861 | index: i, n_rx_rings: adapter->num_rx_queues); |
2862 | |
2863 | devRead->rssConfDesc.confVer = 1; |
2864 | devRead->rssConfDesc.confLen = cpu_to_le32(sizeof(*rssConf)); |
2865 | devRead->rssConfDesc.confPA = |
2866 | cpu_to_le64(adapter->rss_conf_pa); |
2867 | } |
2868 | |
2869 | #endif /* VMXNET3_RSS */ |
2870 | |
2871 | /* intr settings */ |
2872 | if (!VMXNET3_VERSION_GE_6(adapter) || |
2873 | !adapter->queuesExtEnabled) { |
2874 | devRead->intrConf.autoMask = adapter->intr.mask_mode == |
2875 | VMXNET3_IMM_AUTO; |
2876 | devRead->intrConf.numIntrs = adapter->intr.num_intrs; |
2877 | for (i = 0; i < adapter->intr.num_intrs; i++) |
2878 | devRead->intrConf.modLevels[i] = adapter->intr.mod_levels[i]; |
2879 | |
2880 | devRead->intrConf.eventIntrIdx = adapter->intr.event_intr_idx; |
2881 | devRead->intrConf.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL); |
2882 | } else { |
2883 | devReadExt->intrConfExt.autoMask = adapter->intr.mask_mode == |
2884 | VMXNET3_IMM_AUTO; |
2885 | devReadExt->intrConfExt.numIntrs = adapter->intr.num_intrs; |
2886 | for (i = 0; i < adapter->intr.num_intrs; i++) |
2887 | devReadExt->intrConfExt.modLevels[i] = adapter->intr.mod_levels[i]; |
2888 | |
2889 | devReadExt->intrConfExt.eventIntrIdx = adapter->intr.event_intr_idx; |
2890 | devReadExt->intrConfExt.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL); |
2891 | } |
2892 | |
2893 | /* rx filter settings */ |
2894 | devRead->rxFilterConf.rxMode = 0; |
2895 | vmxnet3_restore_vlan(adapter); |
2896 | vmxnet3_write_mac_addr(adapter, mac: adapter->netdev->dev_addr); |
2897 | |
2898 | /* the rest are already zeroed */ |
2899 | } |
2900 | |
2901 | static void |
2902 | vmxnet3_init_bufsize(struct vmxnet3_adapter *adapter) |
2903 | { |
2904 | struct Vmxnet3_DriverShared *shared = adapter->shared; |
2905 | union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo; |
2906 | unsigned long flags; |
2907 | |
2908 | if (!VMXNET3_VERSION_GE_7(adapter)) |
2909 | return; |
2910 | |
2911 | cmdInfo->ringBufSize = adapter->ringBufSize; |
2912 | spin_lock_irqsave(&adapter->cmd_lock, flags); |
2913 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, |
2914 | VMXNET3_CMD_SET_RING_BUFFER_SIZE); |
2915 | spin_unlock_irqrestore(lock: &adapter->cmd_lock, flags); |
2916 | } |
2917 | |
2918 | static void |
2919 | vmxnet3_init_coalesce(struct vmxnet3_adapter *adapter) |
2920 | { |
2921 | struct Vmxnet3_DriverShared *shared = adapter->shared; |
2922 | union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo; |
2923 | unsigned long flags; |
2924 | |
2925 | if (!VMXNET3_VERSION_GE_3(adapter)) |
2926 | return; |
2927 | |
2928 | spin_lock_irqsave(&adapter->cmd_lock, flags); |
2929 | cmdInfo->varConf.confVer = 1; |
2930 | cmdInfo->varConf.confLen = |
2931 | cpu_to_le32(sizeof(*adapter->coal_conf)); |
2932 | cmdInfo->varConf.confPA = cpu_to_le64(adapter->coal_conf_pa); |
2933 | |
2934 | if (adapter->default_coal_mode) { |
2935 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, |
2936 | VMXNET3_CMD_GET_COALESCE); |
2937 | } else { |
2938 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, |
2939 | VMXNET3_CMD_SET_COALESCE); |
2940 | } |
2941 | |
2942 | spin_unlock_irqrestore(lock: &adapter->cmd_lock, flags); |
2943 | } |
2944 | |
2945 | static void |
2946 | (struct vmxnet3_adapter *adapter) |
2947 | { |
2948 | struct Vmxnet3_DriverShared *shared = adapter->shared; |
2949 | union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo; |
2950 | unsigned long flags; |
2951 | |
2952 | if (!VMXNET3_VERSION_GE_4(adapter)) |
2953 | return; |
2954 | |
2955 | spin_lock_irqsave(&adapter->cmd_lock, flags); |
2956 | |
2957 | if (adapter->default_rss_fields) { |
2958 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, |
2959 | VMXNET3_CMD_GET_RSS_FIELDS); |
2960 | adapter->rss_fields = |
2961 | VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); |
2962 | } else { |
2963 | if (VMXNET3_VERSION_GE_7(adapter)) { |
2964 | if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_UDPIP4 || |
2965 | adapter->rss_fields & VMXNET3_RSS_FIELDS_UDPIP6) && |
2966 | vmxnet3_check_ptcapability(cap_supported: adapter->ptcap_supported[0], |
2967 | VMXNET3_CAP_UDP_RSS)) { |
2968 | adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_UDP_RSS; |
2969 | } else { |
2970 | adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_UDP_RSS); |
2971 | } |
2972 | |
2973 | if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_ESPIP4) && |
2974 | vmxnet3_check_ptcapability(cap_supported: adapter->ptcap_supported[0], |
2975 | VMXNET3_CAP_ESP_RSS_IPV4)) { |
2976 | adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_ESP_RSS_IPV4; |
2977 | } else { |
2978 | adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_ESP_RSS_IPV4); |
2979 | } |
2980 | |
2981 | if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_ESPIP6) && |
2982 | vmxnet3_check_ptcapability(cap_supported: adapter->ptcap_supported[0], |
2983 | VMXNET3_CAP_ESP_RSS_IPV6)) { |
2984 | adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_ESP_RSS_IPV6; |
2985 | } else { |
2986 | adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_ESP_RSS_IPV6); |
2987 | } |
2988 | |
2989 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]); |
2990 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); |
2991 | adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); |
2992 | } |
2993 | cmdInfo->setRssFields = adapter->rss_fields; |
2994 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, |
2995 | VMXNET3_CMD_SET_RSS_FIELDS); |
2996 | /* Not all requested RSS may get applied, so get and |
2997 | * cache what was actually applied. |
2998 | */ |
2999 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, |
3000 | VMXNET3_CMD_GET_RSS_FIELDS); |
3001 | adapter->rss_fields = |
3002 | VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); |
3003 | } |
3004 | |
3005 | spin_unlock_irqrestore(lock: &adapter->cmd_lock, flags); |
3006 | } |
3007 | |
3008 | int |
3009 | vmxnet3_activate_dev(struct vmxnet3_adapter *adapter) |
3010 | { |
3011 | int err, i; |
3012 | u32 ret; |
3013 | unsigned long flags; |
3014 | |
3015 | netdev_dbg(adapter->netdev, "%s: skb_buf_size %d, rx_buf_per_pkt %d," |
3016 | " ring sizes %u %u %u\n" , adapter->netdev->name, |
3017 | adapter->skb_buf_size, adapter->rx_buf_per_pkt, |
3018 | adapter->tx_queue[0].tx_ring.size, |
3019 | adapter->rx_queue[0].rx_ring[0].size, |
3020 | adapter->rx_queue[0].rx_ring[1].size); |
3021 | |
3022 | vmxnet3_tq_init_all(adapter); |
3023 | err = vmxnet3_rq_init_all(adapter); |
3024 | if (err) { |
3025 | netdev_err(dev: adapter->netdev, |
3026 | format: "Failed to init rx queue error %d\n" , err); |
3027 | goto rq_err; |
3028 | } |
3029 | |
3030 | err = vmxnet3_request_irqs(adapter); |
3031 | if (err) { |
3032 | netdev_err(dev: adapter->netdev, |
3033 | format: "Failed to setup irq for error %d\n" , err); |
3034 | goto irq_err; |
3035 | } |
3036 | |
3037 | vmxnet3_setup_driver_shared(adapter); |
3038 | |
3039 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, VMXNET3_GET_ADDR_LO( |
3040 | adapter->shared_pa)); |
3041 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, VMXNET3_GET_ADDR_HI( |
3042 | adapter->shared_pa)); |
3043 | spin_lock_irqsave(&adapter->cmd_lock, flags); |
3044 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, |
3045 | VMXNET3_CMD_ACTIVATE_DEV); |
3046 | ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); |
3047 | spin_unlock_irqrestore(lock: &adapter->cmd_lock, flags); |
3048 | |
3049 | if (ret != 0) { |
3050 | netdev_err(dev: adapter->netdev, |
3051 | format: "Failed to activate dev: error %u\n" , ret); |
3052 | err = -EINVAL; |
3053 | goto activate_err; |
3054 | } |
3055 | |
3056 | vmxnet3_init_bufsize(adapter); |
3057 | vmxnet3_init_coalesce(adapter); |
3058 | vmxnet3_init_rssfields(adapter); |
3059 | |
3060 | for (i = 0; i < adapter->num_rx_queues; i++) { |
3061 | VMXNET3_WRITE_BAR0_REG(adapter, |
3062 | adapter->rx_prod_offset + i * VMXNET3_REG_ALIGN, |
3063 | adapter->rx_queue[i].rx_ring[0].next2fill); |
3064 | VMXNET3_WRITE_BAR0_REG(adapter, (adapter->rx_prod2_offset + |
3065 | (i * VMXNET3_REG_ALIGN)), |
3066 | adapter->rx_queue[i].rx_ring[1].next2fill); |
3067 | } |
3068 | |
3069 | /* Apply the rx filter settins last. */ |
3070 | vmxnet3_set_mc(netdev: adapter->netdev); |
3071 | |
3072 | /* |
3073 | * Check link state when first activating device. It will start the |
3074 | * tx queue if the link is up. |
3075 | */ |
3076 | vmxnet3_check_link(adapter, affectTxQueue: true); |
3077 | netif_tx_wake_all_queues(dev: adapter->netdev); |
3078 | for (i = 0; i < adapter->num_rx_queues; i++) |
3079 | napi_enable(n: &adapter->rx_queue[i].napi); |
3080 | vmxnet3_enable_all_intrs(adapter); |
3081 | clear_bit(VMXNET3_STATE_BIT_QUIESCED, addr: &adapter->state); |
3082 | return 0; |
3083 | |
3084 | activate_err: |
3085 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, 0); |
3086 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, 0); |
3087 | vmxnet3_free_irqs(adapter); |
3088 | irq_err: |
3089 | rq_err: |
3090 | /* free up buffers we allocated */ |
3091 | vmxnet3_rq_cleanup_all(adapter); |
3092 | return err; |
3093 | } |
3094 | |
3095 | |
3096 | void |
3097 | vmxnet3_reset_dev(struct vmxnet3_adapter *adapter) |
3098 | { |
3099 | unsigned long flags; |
3100 | spin_lock_irqsave(&adapter->cmd_lock, flags); |
3101 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV); |
3102 | spin_unlock_irqrestore(lock: &adapter->cmd_lock, flags); |
3103 | } |
3104 | |
3105 | |
3106 | int |
3107 | vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter) |
3108 | { |
3109 | int i; |
3110 | unsigned long flags; |
3111 | if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, addr: &adapter->state)) |
3112 | return 0; |
3113 | |
3114 | |
3115 | spin_lock_irqsave(&adapter->cmd_lock, flags); |
3116 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, |
3117 | VMXNET3_CMD_QUIESCE_DEV); |
3118 | spin_unlock_irqrestore(lock: &adapter->cmd_lock, flags); |
3119 | vmxnet3_disable_all_intrs(adapter); |
3120 | |
3121 | for (i = 0; i < adapter->num_rx_queues; i++) |
3122 | napi_disable(n: &adapter->rx_queue[i].napi); |
3123 | netif_tx_disable(dev: adapter->netdev); |
3124 | adapter->link_speed = 0; |
3125 | netif_carrier_off(dev: adapter->netdev); |
3126 | |
3127 | vmxnet3_tq_cleanup_all(adapter); |
3128 | vmxnet3_rq_cleanup_all(adapter); |
3129 | vmxnet3_free_irqs(adapter); |
3130 | return 0; |
3131 | } |
3132 | |
3133 | |
3134 | static void |
3135 | vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, const u8 *mac) |
3136 | { |
3137 | u32 tmp; |
3138 | |
3139 | tmp = *(u32 *)mac; |
3140 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACL, tmp); |
3141 | |
3142 | tmp = (mac[5] << 8) | mac[4]; |
3143 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACH, tmp); |
3144 | } |
3145 | |
3146 | |
3147 | static int |
3148 | vmxnet3_set_mac_addr(struct net_device *netdev, void *p) |
3149 | { |
3150 | struct sockaddr *addr = p; |
3151 | struct vmxnet3_adapter *adapter = netdev_priv(dev: netdev); |
3152 | |
3153 | dev_addr_set(dev: netdev, addr: addr->sa_data); |
3154 | vmxnet3_write_mac_addr(adapter, mac: addr->sa_data); |
3155 | |
3156 | return 0; |
3157 | } |
3158 | |
3159 | |
3160 | /* ==================== initialization and cleanup routines ============ */ |
3161 | |
3162 | static int |
3163 | vmxnet3_alloc_pci_resources(struct vmxnet3_adapter *adapter) |
3164 | { |
3165 | int err; |
3166 | unsigned long mmio_start, mmio_len; |
3167 | struct pci_dev *pdev = adapter->pdev; |
3168 | |
3169 | err = pci_enable_device(dev: pdev); |
3170 | if (err) { |
3171 | dev_err(&pdev->dev, "Failed to enable adapter: error %d\n" , err); |
3172 | return err; |
3173 | } |
3174 | |
3175 | err = pci_request_selected_regions(pdev, (1 << 2) - 1, |
3176 | vmxnet3_driver_name); |
3177 | if (err) { |
3178 | dev_err(&pdev->dev, |
3179 | "Failed to request region for adapter: error %d\n" , err); |
3180 | goto err_enable_device; |
3181 | } |
3182 | |
3183 | pci_set_master(dev: pdev); |
3184 | |
3185 | mmio_start = pci_resource_start(pdev, 0); |
3186 | mmio_len = pci_resource_len(pdev, 0); |
3187 | adapter->hw_addr0 = ioremap(offset: mmio_start, size: mmio_len); |
3188 | if (!adapter->hw_addr0) { |
3189 | dev_err(&pdev->dev, "Failed to map bar0\n" ); |
3190 | err = -EIO; |
3191 | goto err_ioremap; |
3192 | } |
3193 | |
3194 | mmio_start = pci_resource_start(pdev, 1); |
3195 | mmio_len = pci_resource_len(pdev, 1); |
3196 | adapter->hw_addr1 = ioremap(offset: mmio_start, size: mmio_len); |
3197 | if (!adapter->hw_addr1) { |
3198 | dev_err(&pdev->dev, "Failed to map bar1\n" ); |
3199 | err = -EIO; |
3200 | goto err_bar1; |
3201 | } |
3202 | return 0; |
3203 | |
3204 | err_bar1: |
3205 | iounmap(addr: adapter->hw_addr0); |
3206 | err_ioremap: |
3207 | pci_release_selected_regions(pdev, (1 << 2) - 1); |
3208 | err_enable_device: |
3209 | pci_disable_device(dev: pdev); |
3210 | return err; |
3211 | } |
3212 | |
3213 | |
3214 | static void |
3215 | vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter) |
3216 | { |
3217 | BUG_ON(!adapter->pdev); |
3218 | |
3219 | iounmap(addr: adapter->hw_addr0); |
3220 | iounmap(addr: adapter->hw_addr1); |
3221 | pci_release_selected_regions(adapter->pdev, (1 << 2) - 1); |
3222 | pci_disable_device(dev: adapter->pdev); |
3223 | } |
3224 | |
3225 | |
3226 | void |
3227 | vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter) |
3228 | { |
3229 | size_t sz, i, ring0_size, ring1_size, comp_size; |
3230 | /* With version7 ring1 will have only T0 buffers */ |
3231 | if (!VMXNET3_VERSION_GE_7(adapter)) { |
3232 | if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE - |
3233 | VMXNET3_MAX_ETH_HDR_SIZE) { |
3234 | adapter->skb_buf_size = adapter->netdev->mtu + |
3235 | VMXNET3_MAX_ETH_HDR_SIZE; |
3236 | if (adapter->skb_buf_size < VMXNET3_MIN_T0_BUF_SIZE) |
3237 | adapter->skb_buf_size = VMXNET3_MIN_T0_BUF_SIZE; |
3238 | |
3239 | adapter->rx_buf_per_pkt = 1; |
3240 | } else { |
3241 | adapter->skb_buf_size = VMXNET3_MAX_SKB_BUF_SIZE; |
3242 | sz = adapter->netdev->mtu - VMXNET3_MAX_SKB_BUF_SIZE + |
3243 | VMXNET3_MAX_ETH_HDR_SIZE; |
3244 | adapter->rx_buf_per_pkt = 1 + (sz + PAGE_SIZE - 1) / PAGE_SIZE; |
3245 | } |
3246 | } else { |
3247 | adapter->skb_buf_size = min((int)adapter->netdev->mtu + VMXNET3_MAX_ETH_HDR_SIZE, |
3248 | VMXNET3_MAX_SKB_BUF_SIZE); |
3249 | adapter->rx_buf_per_pkt = 1; |
3250 | adapter->ringBufSize.ring1BufSizeType0 = cpu_to_le16(adapter->skb_buf_size); |
3251 | adapter->ringBufSize.ring1BufSizeType1 = 0; |
3252 | adapter->ringBufSize.ring2BufSizeType1 = cpu_to_le16(PAGE_SIZE); |
3253 | } |
3254 | |
3255 | /* |
3256 | * for simplicity, force the ring0 size to be a multiple of |
3257 | * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN |
3258 | */ |
3259 | sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN; |
3260 | ring0_size = adapter->rx_queue[0].rx_ring[0].size; |
3261 | ring0_size = (ring0_size + sz - 1) / sz * sz; |
3262 | ring0_size = min_t(u32, ring0_size, VMXNET3_RX_RING_MAX_SIZE / |
3263 | sz * sz); |
3264 | ring1_size = adapter->rx_queue[0].rx_ring[1].size; |
3265 | ring1_size = (ring1_size + sz - 1) / sz * sz; |
3266 | ring1_size = min_t(u32, ring1_size, VMXNET3_RX_RING2_MAX_SIZE / |
3267 | sz * sz); |
3268 | /* For v7 and later, keep ring size power of 2 for UPT */ |
3269 | if (VMXNET3_VERSION_GE_7(adapter)) { |
3270 | ring0_size = rounddown_pow_of_two(ring0_size); |
3271 | ring1_size = rounddown_pow_of_two(ring1_size); |
3272 | } |
3273 | comp_size = ring0_size + ring1_size; |
3274 | |
3275 | for (i = 0; i < adapter->num_rx_queues; i++) { |
3276 | struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; |
3277 | |
3278 | rq->rx_ring[0].size = ring0_size; |
3279 | rq->rx_ring[1].size = ring1_size; |
3280 | rq->comp_ring.size = comp_size; |
3281 | } |
3282 | } |
3283 | |
3284 | |
3285 | int |
3286 | vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size, |
3287 | u32 rx_ring_size, u32 rx_ring2_size, |
3288 | u16 txdata_desc_size, u16 rxdata_desc_size) |
3289 | { |
3290 | int err = 0, i; |
3291 | |
3292 | for (i = 0; i < adapter->num_tx_queues; i++) { |
3293 | struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i]; |
3294 | tq->tx_ring.size = tx_ring_size; |
3295 | tq->data_ring.size = tx_ring_size; |
3296 | tq->comp_ring.size = tx_ring_size; |
3297 | tq->txdata_desc_size = txdata_desc_size; |
3298 | tq->shared = &adapter->tqd_start[i].ctrl; |
3299 | tq->stopped = true; |
3300 | tq->adapter = adapter; |
3301 | tq->qid = i; |
3302 | err = vmxnet3_tq_create(tq, adapter); |
3303 | /* |
3304 | * Too late to change num_tx_queues. We cannot do away with |
3305 | * lesser number of queues than what we asked for |
3306 | */ |
3307 | if (err) |
3308 | goto queue_err; |
3309 | } |
3310 | |
3311 | adapter->rx_queue[0].rx_ring[0].size = rx_ring_size; |
3312 | adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size; |
3313 | vmxnet3_adjust_rx_ring_size(adapter); |
3314 | |
3315 | adapter->rxdataring_enabled = VMXNET3_VERSION_GE_3(adapter); |
3316 | for (i = 0; i < adapter->num_rx_queues; i++) { |
3317 | struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; |
3318 | /* qid and qid2 for rx queues will be assigned later when num |
3319 | * of rx queues is finalized after allocating intrs */ |
3320 | rq->shared = &adapter->rqd_start[i].ctrl; |
3321 | rq->adapter = adapter; |
3322 | rq->data_ring.desc_size = rxdata_desc_size; |
3323 | err = vmxnet3_rq_create(rq, adapter); |
3324 | if (err) { |
3325 | if (i == 0) { |
3326 | netdev_err(dev: adapter->netdev, |
3327 | format: "Could not allocate any rx queues. " |
3328 | "Aborting.\n" ); |
3329 | goto queue_err; |
3330 | } else { |
3331 | netdev_info(dev: adapter->netdev, |
3332 | format: "Number of rx queues changed " |
3333 | "to : %d.\n" , i); |
3334 | adapter->num_rx_queues = i; |
3335 | err = 0; |
3336 | break; |
3337 | } |
3338 | } |
3339 | } |
3340 | |
3341 | if (!adapter->rxdataring_enabled) |
3342 | vmxnet3_rq_destroy_all_rxdataring(adapter); |
3343 | |
3344 | return err; |
3345 | queue_err: |
3346 | vmxnet3_tq_destroy_all(adapter); |
3347 | return err; |
3348 | } |
3349 | |
3350 | static int |
3351 | vmxnet3_open(struct net_device *netdev) |
3352 | { |
3353 | struct vmxnet3_adapter *adapter; |
3354 | int err, i; |
3355 | |
3356 | adapter = netdev_priv(dev: netdev); |
3357 | |
3358 | for (i = 0; i < adapter->num_tx_queues; i++) |
3359 | spin_lock_init(&adapter->tx_queue[i].tx_lock); |
3360 | |
3361 | if (VMXNET3_VERSION_GE_3(adapter)) { |
3362 | unsigned long flags; |
3363 | u16 txdata_desc_size; |
3364 | |
3365 | spin_lock_irqsave(&adapter->cmd_lock, flags); |
3366 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, |
3367 | VMXNET3_CMD_GET_TXDATA_DESC_SIZE); |
3368 | txdata_desc_size = VMXNET3_READ_BAR1_REG(adapter, |
3369 | VMXNET3_REG_CMD); |
3370 | spin_unlock_irqrestore(lock: &adapter->cmd_lock, flags); |
3371 | |
3372 | if ((txdata_desc_size < VMXNET3_TXDATA_DESC_MIN_SIZE) || |
3373 | (txdata_desc_size > VMXNET3_TXDATA_DESC_MAX_SIZE) || |
3374 | (txdata_desc_size & VMXNET3_TXDATA_DESC_SIZE_MASK)) { |
3375 | adapter->txdata_desc_size = |
3376 | sizeof(struct Vmxnet3_TxDataDesc); |
3377 | } else { |
3378 | adapter->txdata_desc_size = txdata_desc_size; |
3379 | } |
3380 | } else { |
3381 | adapter->txdata_desc_size = sizeof(struct Vmxnet3_TxDataDesc); |
3382 | } |
3383 | |
3384 | err = vmxnet3_create_queues(adapter, |
3385 | tx_ring_size: adapter->tx_ring_size, |
3386 | rx_ring_size: adapter->rx_ring_size, |
3387 | rx_ring2_size: adapter->rx_ring2_size, |
3388 | txdata_desc_size: adapter->txdata_desc_size, |
3389 | rxdata_desc_size: adapter->rxdata_desc_size); |
3390 | if (err) |
3391 | goto queue_err; |
3392 | |
3393 | err = vmxnet3_activate_dev(adapter); |
3394 | if (err) |
3395 | goto activate_err; |
3396 | |
3397 | return 0; |
3398 | |
3399 | activate_err: |
3400 | vmxnet3_rq_destroy_all(adapter); |
3401 | vmxnet3_tq_destroy_all(adapter); |
3402 | queue_err: |
3403 | return err; |
3404 | } |
3405 | |
3406 | |
3407 | static int |
3408 | vmxnet3_close(struct net_device *netdev) |
3409 | { |
3410 | struct vmxnet3_adapter *adapter = netdev_priv(dev: netdev); |
3411 | |
3412 | /* |
3413 | * Reset_work may be in the middle of resetting the device, wait for its |
3414 | * completion. |
3415 | */ |
3416 | while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, addr: &adapter->state)) |
3417 | usleep_range(min: 1000, max: 2000); |
3418 | |
3419 | vmxnet3_quiesce_dev(adapter); |
3420 | |
3421 | vmxnet3_rq_destroy_all(adapter); |
3422 | vmxnet3_tq_destroy_all(adapter); |
3423 | |
3424 | clear_bit(VMXNET3_STATE_BIT_RESETTING, addr: &adapter->state); |
3425 | |
3426 | |
3427 | return 0; |
3428 | } |
3429 | |
3430 | |
3431 | void |
3432 | vmxnet3_force_close(struct vmxnet3_adapter *adapter) |
3433 | { |
3434 | int i; |
3435 | |
3436 | /* |
3437 | * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise |
3438 | * vmxnet3_close() will deadlock. |
3439 | */ |
3440 | BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)); |
3441 | |
3442 | /* we need to enable NAPI, otherwise dev_close will deadlock */ |
3443 | for (i = 0; i < adapter->num_rx_queues; i++) |
3444 | napi_enable(n: &adapter->rx_queue[i].napi); |
3445 | /* |
3446 | * Need to clear the quiesce bit to ensure that vmxnet3_close |
3447 | * can quiesce the device properly |
3448 | */ |
3449 | clear_bit(VMXNET3_STATE_BIT_QUIESCED, addr: &adapter->state); |
3450 | dev_close(dev: adapter->netdev); |
3451 | } |
3452 | |
3453 | |
3454 | static int |
3455 | vmxnet3_change_mtu(struct net_device *netdev, int new_mtu) |
3456 | { |
3457 | struct vmxnet3_adapter *adapter = netdev_priv(dev: netdev); |
3458 | int err = 0; |
3459 | |
3460 | netdev->mtu = new_mtu; |
3461 | |
3462 | /* |
3463 | * Reset_work may be in the middle of resetting the device, wait for its |
3464 | * completion. |
3465 | */ |
3466 | while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, addr: &adapter->state)) |
3467 | usleep_range(min: 1000, max: 2000); |
3468 | |
3469 | if (netif_running(dev: netdev)) { |
3470 | vmxnet3_quiesce_dev(adapter); |
3471 | vmxnet3_reset_dev(adapter); |
3472 | |
3473 | /* we need to re-create the rx queue based on the new mtu */ |
3474 | vmxnet3_rq_destroy_all(adapter); |
3475 | vmxnet3_adjust_rx_ring_size(adapter); |
3476 | err = vmxnet3_rq_create_all(adapter); |
3477 | if (err) { |
3478 | netdev_err(dev: netdev, |
3479 | format: "failed to re-create rx queues, " |
3480 | " error %d. Closing it.\n" , err); |
3481 | goto out; |
3482 | } |
3483 | |
3484 | err = vmxnet3_activate_dev(adapter); |
3485 | if (err) { |
3486 | netdev_err(dev: netdev, |
3487 | format: "failed to re-activate, error %d. " |
3488 | "Closing it\n" , err); |
3489 | goto out; |
3490 | } |
3491 | } |
3492 | |
3493 | out: |
3494 | clear_bit(VMXNET3_STATE_BIT_RESETTING, addr: &adapter->state); |
3495 | if (err) |
3496 | vmxnet3_force_close(adapter); |
3497 | |
3498 | return err; |
3499 | } |
3500 | |
3501 | |
3502 | static void |
3503 | vmxnet3_declare_features(struct vmxnet3_adapter *adapter) |
3504 | { |
3505 | struct net_device *netdev = adapter->netdev; |
3506 | |
3507 | netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM | |
3508 | NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX | |
3509 | NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 | |
3510 | NETIF_F_LRO | NETIF_F_HIGHDMA; |
3511 | |
3512 | if (VMXNET3_VERSION_GE_4(adapter)) { |
3513 | netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL | |
3514 | NETIF_F_GSO_UDP_TUNNEL_CSUM; |
3515 | |
3516 | netdev->hw_enc_features = NETIF_F_SG | NETIF_F_RXCSUM | |
3517 | NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX | |
3518 | NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 | |
3519 | NETIF_F_LRO | NETIF_F_GSO_UDP_TUNNEL | |
3520 | NETIF_F_GSO_UDP_TUNNEL_CSUM; |
3521 | } |
3522 | |
3523 | if (VMXNET3_VERSION_GE_7(adapter)) { |
3524 | unsigned long flags; |
3525 | |
3526 | if (vmxnet3_check_ptcapability(cap_supported: adapter->ptcap_supported[0], |
3527 | VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD)) { |
3528 | adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD; |
3529 | } |
3530 | if (vmxnet3_check_ptcapability(cap_supported: adapter->ptcap_supported[0], |
3531 | VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD)) { |
3532 | adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD; |
3533 | } |
3534 | if (vmxnet3_check_ptcapability(cap_supported: adapter->ptcap_supported[0], |
3535 | VMXNET3_CAP_GENEVE_TSO)) { |
3536 | adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_TSO; |
3537 | } |
3538 | if (vmxnet3_check_ptcapability(cap_supported: adapter->ptcap_supported[0], |
3539 | VMXNET3_CAP_VXLAN_TSO)) { |
3540 | adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_TSO; |
3541 | } |
3542 | if (vmxnet3_check_ptcapability(cap_supported: adapter->ptcap_supported[0], |
3543 | VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD)) { |
3544 | adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD; |
3545 | } |
3546 | if (vmxnet3_check_ptcapability(cap_supported: adapter->ptcap_supported[0], |
3547 | VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD)) { |
3548 | adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD; |
3549 | } |
3550 | |
3551 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]); |
3552 | spin_lock_irqsave(&adapter->cmd_lock, flags); |
3553 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); |
3554 | adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); |
3555 | spin_unlock_irqrestore(lock: &adapter->cmd_lock, flags); |
3556 | |
3557 | if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD)) && |
3558 | !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD)) && |
3559 | !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_TSO)) && |
3560 | !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_TSO))) { |
3561 | netdev->hw_enc_features &= ~NETIF_F_GSO_UDP_TUNNEL; |
3562 | netdev->hw_features &= ~NETIF_F_GSO_UDP_TUNNEL; |
3563 | } |
3564 | if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD)) && |
3565 | !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD))) { |
3566 | netdev->hw_enc_features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM; |
3567 | netdev->hw_features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM; |
3568 | } |
3569 | } |
3570 | |
3571 | netdev->vlan_features = netdev->hw_features & |
3572 | ~(NETIF_F_HW_VLAN_CTAG_TX | |
3573 | NETIF_F_HW_VLAN_CTAG_RX); |
3574 | netdev->features = netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; |
3575 | } |
3576 | |
3577 | |
3578 | static void |
3579 | vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac) |
3580 | { |
3581 | u32 tmp; |
3582 | |
3583 | tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL); |
3584 | *(u32 *)mac = tmp; |
3585 | |
3586 | tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH); |
3587 | mac[4] = tmp & 0xff; |
3588 | mac[5] = (tmp >> 8) & 0xff; |
3589 | } |
3590 | |
3591 | #ifdef CONFIG_PCI_MSI |
3592 | |
3593 | /* |
3594 | * Enable MSIx vectors. |
3595 | * Returns : |
3596 | * VMXNET3_LINUX_MIN_MSIX_VECT when only minimum number of vectors required |
3597 | * were enabled. |
3598 | * number of vectors which were enabled otherwise (this number is greater |
3599 | * than VMXNET3_LINUX_MIN_MSIX_VECT) |
3600 | */ |
3601 | |
3602 | static int |
3603 | vmxnet3_acquire_msix_vectors(struct vmxnet3_adapter *adapter, int nvec) |
3604 | { |
3605 | int ret = pci_enable_msix_range(dev: adapter->pdev, |
3606 | entries: adapter->intr.msix_entries, minvec: nvec, maxvec: nvec); |
3607 | |
3608 | if (ret == -ENOSPC && nvec > VMXNET3_LINUX_MIN_MSIX_VECT) { |
3609 | dev_err(&adapter->netdev->dev, |
3610 | "Failed to enable %d MSI-X, trying %d\n" , |
3611 | nvec, VMXNET3_LINUX_MIN_MSIX_VECT); |
3612 | |
3613 | ret = pci_enable_msix_range(dev: adapter->pdev, |
3614 | entries: adapter->intr.msix_entries, |
3615 | VMXNET3_LINUX_MIN_MSIX_VECT, |
3616 | VMXNET3_LINUX_MIN_MSIX_VECT); |
3617 | } |
3618 | |
3619 | if (ret < 0) { |
3620 | dev_err(&adapter->netdev->dev, |
3621 | "Failed to enable MSI-X, error: %d\n" , ret); |
3622 | } |
3623 | |
3624 | return ret; |
3625 | } |
3626 | |
3627 | |
3628 | #endif /* CONFIG_PCI_MSI */ |
3629 | |
3630 | static void |
3631 | vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter) |
3632 | { |
3633 | u32 cfg; |
3634 | unsigned long flags; |
3635 | |
3636 | /* intr settings */ |
3637 | spin_lock_irqsave(&adapter->cmd_lock, flags); |
3638 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, |
3639 | VMXNET3_CMD_GET_CONF_INTR); |
3640 | cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); |
3641 | spin_unlock_irqrestore(lock: &adapter->cmd_lock, flags); |
3642 | adapter->intr.type = cfg & 0x3; |
3643 | adapter->intr.mask_mode = (cfg >> 2) & 0x3; |
3644 | |
3645 | if (adapter->intr.type == VMXNET3_IT_AUTO) { |
3646 | adapter->intr.type = VMXNET3_IT_MSIX; |
3647 | } |
3648 | |
3649 | #ifdef CONFIG_PCI_MSI |
3650 | if (adapter->intr.type == VMXNET3_IT_MSIX) { |
3651 | int i, nvec, nvec_allocated; |
3652 | |
3653 | nvec = adapter->share_intr == VMXNET3_INTR_TXSHARE ? |
3654 | 1 : adapter->num_tx_queues; |
3655 | nvec += adapter->share_intr == VMXNET3_INTR_BUDDYSHARE ? |
3656 | 0 : adapter->num_rx_queues; |
3657 | nvec += 1; /* for link event */ |
3658 | nvec = nvec > VMXNET3_LINUX_MIN_MSIX_VECT ? |
3659 | nvec : VMXNET3_LINUX_MIN_MSIX_VECT; |
3660 | |
3661 | for (i = 0; i < nvec; i++) |
3662 | adapter->intr.msix_entries[i].entry = i; |
3663 | |
3664 | nvec_allocated = vmxnet3_acquire_msix_vectors(adapter, nvec); |
3665 | if (nvec_allocated < 0) |
3666 | goto msix_err; |
3667 | |
3668 | /* If we cannot allocate one MSIx vector per queue |
3669 | * then limit the number of rx queues to 1 |
3670 | */ |
3671 | if (nvec_allocated == VMXNET3_LINUX_MIN_MSIX_VECT && |
3672 | nvec != VMXNET3_LINUX_MIN_MSIX_VECT) { |
3673 | if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE |
3674 | || adapter->num_rx_queues != 1) { |
3675 | adapter->share_intr = VMXNET3_INTR_TXSHARE; |
3676 | netdev_err(dev: adapter->netdev, |
3677 | format: "Number of rx queues : 1\n" ); |
3678 | adapter->num_rx_queues = 1; |
3679 | } |
3680 | } |
3681 | |
3682 | adapter->intr.num_intrs = nvec_allocated; |
3683 | return; |
3684 | |
3685 | msix_err: |
3686 | /* If we cannot allocate MSIx vectors use only one rx queue */ |
3687 | dev_info(&adapter->pdev->dev, |
3688 | "Failed to enable MSI-X, error %d. " |
3689 | "Limiting #rx queues to 1, try MSI.\n" , nvec_allocated); |
3690 | |
3691 | adapter->intr.type = VMXNET3_IT_MSI; |
3692 | } |
3693 | |
3694 | if (adapter->intr.type == VMXNET3_IT_MSI) { |
3695 | if (!pci_enable_msi(dev: adapter->pdev)) { |
3696 | adapter->num_rx_queues = 1; |
3697 | adapter->intr.num_intrs = 1; |
3698 | return; |
3699 | } |
3700 | } |
3701 | #endif /* CONFIG_PCI_MSI */ |
3702 | |
3703 | adapter->num_rx_queues = 1; |
3704 | dev_info(&adapter->netdev->dev, |
3705 | "Using INTx interrupt, #Rx queues: 1.\n" ); |
3706 | adapter->intr.type = VMXNET3_IT_INTX; |
3707 | |
3708 | /* INT-X related setting */ |
3709 | adapter->intr.num_intrs = 1; |
3710 | } |
3711 | |
3712 | |
3713 | static void |
3714 | vmxnet3_free_intr_resources(struct vmxnet3_adapter *adapter) |
3715 | { |
3716 | if (adapter->intr.type == VMXNET3_IT_MSIX) |
3717 | pci_disable_msix(dev: adapter->pdev); |
3718 | else if (adapter->intr.type == VMXNET3_IT_MSI) |
3719 | pci_disable_msi(dev: adapter->pdev); |
3720 | else |
3721 | BUG_ON(adapter->intr.type != VMXNET3_IT_INTX); |
3722 | } |
3723 | |
3724 | |
3725 | static void |
3726 | vmxnet3_tx_timeout(struct net_device *netdev, unsigned int txqueue) |
3727 | { |
3728 | struct vmxnet3_adapter *adapter = netdev_priv(dev: netdev); |
3729 | adapter->tx_timeout_count++; |
3730 | |
3731 | netdev_err(dev: adapter->netdev, format: "tx hang\n" ); |
3732 | schedule_work(work: &adapter->work); |
3733 | } |
3734 | |
3735 | |
3736 | static void |
3737 | vmxnet3_reset_work(struct work_struct *data) |
3738 | { |
3739 | struct vmxnet3_adapter *adapter; |
3740 | |
3741 | adapter = container_of(data, struct vmxnet3_adapter, work); |
3742 | |
3743 | /* if another thread is resetting the device, no need to proceed */ |
3744 | if (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, addr: &adapter->state)) |
3745 | return; |
3746 | |
3747 | /* if the device is closed, we must leave it alone */ |
3748 | rtnl_lock(); |
3749 | if (netif_running(dev: adapter->netdev)) { |
3750 | netdev_notice(dev: adapter->netdev, format: "resetting\n" ); |
3751 | vmxnet3_quiesce_dev(adapter); |
3752 | vmxnet3_reset_dev(adapter); |
3753 | vmxnet3_activate_dev(adapter); |
3754 | } else { |
3755 | netdev_info(dev: adapter->netdev, format: "already closed\n" ); |
3756 | } |
3757 | rtnl_unlock(); |
3758 | |
3759 | netif_wake_queue(dev: adapter->netdev); |
3760 | clear_bit(VMXNET3_STATE_BIT_RESETTING, addr: &adapter->state); |
3761 | } |
3762 | |
3763 | |
3764 | static int |
3765 | vmxnet3_probe_device(struct pci_dev *pdev, |
3766 | const struct pci_device_id *id) |
3767 | { |
3768 | static const struct net_device_ops vmxnet3_netdev_ops = { |
3769 | .ndo_open = vmxnet3_open, |
3770 | .ndo_stop = vmxnet3_close, |
3771 | .ndo_start_xmit = vmxnet3_xmit_frame, |
3772 | .ndo_set_mac_address = vmxnet3_set_mac_addr, |
3773 | .ndo_change_mtu = vmxnet3_change_mtu, |
3774 | .ndo_fix_features = vmxnet3_fix_features, |
3775 | .ndo_set_features = vmxnet3_set_features, |
3776 | .ndo_features_check = vmxnet3_features_check, |
3777 | .ndo_get_stats64 = vmxnet3_get_stats64, |
3778 | .ndo_tx_timeout = vmxnet3_tx_timeout, |
3779 | .ndo_set_rx_mode = vmxnet3_set_mc, |
3780 | .ndo_vlan_rx_add_vid = vmxnet3_vlan_rx_add_vid, |
3781 | .ndo_vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid, |
3782 | #ifdef CONFIG_NET_POLL_CONTROLLER |
3783 | .ndo_poll_controller = vmxnet3_netpoll, |
3784 | #endif |
3785 | .ndo_bpf = vmxnet3_xdp, |
3786 | .ndo_xdp_xmit = vmxnet3_xdp_xmit, |
3787 | }; |
3788 | int err; |
3789 | u32 ver; |
3790 | struct net_device *netdev; |
3791 | struct vmxnet3_adapter *adapter; |
3792 | u8 mac[ETH_ALEN]; |
3793 | int size; |
3794 | int num_tx_queues; |
3795 | int num_rx_queues; |
3796 | int queues; |
3797 | unsigned long flags; |
3798 | |
3799 | if (!pci_msi_enabled()) |
3800 | enable_mq = 0; |
3801 | |
3802 | #ifdef VMXNET3_RSS |
3803 | if (enable_mq) |
3804 | num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES, |
3805 | (int)num_online_cpus()); |
3806 | else |
3807 | #endif |
3808 | num_rx_queues = 1; |
3809 | |
3810 | if (enable_mq) |
3811 | num_tx_queues = min(VMXNET3_DEVICE_MAX_TX_QUEUES, |
3812 | (int)num_online_cpus()); |
3813 | else |
3814 | num_tx_queues = 1; |
3815 | |
3816 | netdev = alloc_etherdev_mq(sizeof(struct vmxnet3_adapter), |
3817 | max(num_tx_queues, num_rx_queues)); |
3818 | if (!netdev) |
3819 | return -ENOMEM; |
3820 | |
3821 | pci_set_drvdata(pdev, data: netdev); |
3822 | adapter = netdev_priv(dev: netdev); |
3823 | adapter->netdev = netdev; |
3824 | adapter->pdev = pdev; |
3825 | |
3826 | adapter->tx_ring_size = VMXNET3_DEF_TX_RING_SIZE; |
3827 | adapter->rx_ring_size = VMXNET3_DEF_RX_RING_SIZE; |
3828 | adapter->rx_ring2_size = VMXNET3_DEF_RX_RING2_SIZE; |
3829 | |
3830 | err = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(64)); |
3831 | if (err) { |
3832 | dev_err(&pdev->dev, "dma_set_mask failed\n" ); |
3833 | goto err_set_mask; |
3834 | } |
3835 | |
3836 | spin_lock_init(&adapter->cmd_lock); |
3837 | adapter->adapter_pa = dma_map_single(&adapter->pdev->dev, adapter, |
3838 | sizeof(struct vmxnet3_adapter), |
3839 | DMA_TO_DEVICE); |
3840 | if (dma_mapping_error(dev: &adapter->pdev->dev, dma_addr: adapter->adapter_pa)) { |
3841 | dev_err(&pdev->dev, "Failed to map dma\n" ); |
3842 | err = -EFAULT; |
3843 | goto err_set_mask; |
3844 | } |
3845 | adapter->shared = dma_alloc_coherent( |
3846 | dev: &adapter->pdev->dev, |
3847 | size: sizeof(struct Vmxnet3_DriverShared), |
3848 | dma_handle: &adapter->shared_pa, GFP_KERNEL); |
3849 | if (!adapter->shared) { |
3850 | dev_err(&pdev->dev, "Failed to allocate memory\n" ); |
3851 | err = -ENOMEM; |
3852 | goto err_alloc_shared; |
3853 | } |
3854 | |
3855 | err = vmxnet3_alloc_pci_resources(adapter); |
3856 | if (err < 0) |
3857 | goto err_alloc_pci; |
3858 | |
3859 | ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS); |
3860 | if (ver & (1 << VMXNET3_REV_7)) { |
3861 | VMXNET3_WRITE_BAR1_REG(adapter, |
3862 | VMXNET3_REG_VRRS, |
3863 | 1 << VMXNET3_REV_7); |
3864 | adapter->version = VMXNET3_REV_7 + 1; |
3865 | } else if (ver & (1 << VMXNET3_REV_6)) { |
3866 | VMXNET3_WRITE_BAR1_REG(adapter, |
3867 | VMXNET3_REG_VRRS, |
3868 | 1 << VMXNET3_REV_6); |
3869 | adapter->version = VMXNET3_REV_6 + 1; |
3870 | } else if (ver & (1 << VMXNET3_REV_5)) { |
3871 | VMXNET3_WRITE_BAR1_REG(adapter, |
3872 | VMXNET3_REG_VRRS, |
3873 | 1 << VMXNET3_REV_5); |
3874 | adapter->version = VMXNET3_REV_5 + 1; |
3875 | } else if (ver & (1 << VMXNET3_REV_4)) { |
3876 | VMXNET3_WRITE_BAR1_REG(adapter, |
3877 | VMXNET3_REG_VRRS, |
3878 | 1 << VMXNET3_REV_4); |
3879 | adapter->version = VMXNET3_REV_4 + 1; |
3880 | } else if (ver & (1 << VMXNET3_REV_3)) { |
3881 | VMXNET3_WRITE_BAR1_REG(adapter, |
3882 | VMXNET3_REG_VRRS, |
3883 | 1 << VMXNET3_REV_3); |
3884 | adapter->version = VMXNET3_REV_3 + 1; |
3885 | } else if (ver & (1 << VMXNET3_REV_2)) { |
3886 | VMXNET3_WRITE_BAR1_REG(adapter, |
3887 | VMXNET3_REG_VRRS, |
3888 | 1 << VMXNET3_REV_2); |
3889 | adapter->version = VMXNET3_REV_2 + 1; |
3890 | } else if (ver & (1 << VMXNET3_REV_1)) { |
3891 | VMXNET3_WRITE_BAR1_REG(adapter, |
3892 | VMXNET3_REG_VRRS, |
3893 | 1 << VMXNET3_REV_1); |
3894 | adapter->version = VMXNET3_REV_1 + 1; |
3895 | } else { |
3896 | dev_err(&pdev->dev, |
3897 | "Incompatible h/w version (0x%x) for adapter\n" , ver); |
3898 | err = -EBUSY; |
3899 | goto err_ver; |
3900 | } |
3901 | dev_dbg(&pdev->dev, "Using device version %d\n" , adapter->version); |
3902 | |
3903 | ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS); |
3904 | if (ver & 1) { |
3905 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_UVRS, 1); |
3906 | } else { |
3907 | dev_err(&pdev->dev, |
3908 | "Incompatible upt version (0x%x) for adapter\n" , ver); |
3909 | err = -EBUSY; |
3910 | goto err_ver; |
3911 | } |
3912 | |
3913 | if (VMXNET3_VERSION_GE_7(adapter)) { |
3914 | adapter->devcap_supported[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_DCR); |
3915 | adapter->ptcap_supported[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_PTCR); |
3916 | if (adapter->devcap_supported[0] & (1UL << VMXNET3_CAP_LARGE_BAR)) { |
3917 | adapter->dev_caps[0] = adapter->devcap_supported[0] & |
3918 | (1UL << VMXNET3_CAP_LARGE_BAR); |
3919 | } |
3920 | if (!(adapter->ptcap_supported[0] & (1UL << VMXNET3_DCR_ERROR)) && |
3921 | adapter->ptcap_supported[0] & (1UL << VMXNET3_CAP_OOORX_COMP) && |
3922 | adapter->devcap_supported[0] & (1UL << VMXNET3_CAP_OOORX_COMP)) { |
3923 | adapter->dev_caps[0] |= adapter->devcap_supported[0] & |
3924 | (1UL << VMXNET3_CAP_OOORX_COMP); |
3925 | } |
3926 | if (adapter->dev_caps[0]) |
3927 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]); |
3928 | |
3929 | spin_lock_irqsave(&adapter->cmd_lock, flags); |
3930 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); |
3931 | adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); |
3932 | spin_unlock_irqrestore(lock: &adapter->cmd_lock, flags); |
3933 | } |
3934 | |
3935 | if (VMXNET3_VERSION_GE_7(adapter) && |
3936 | adapter->dev_caps[0] & (1UL << VMXNET3_CAP_LARGE_BAR)) { |
3937 | adapter->tx_prod_offset = VMXNET3_REG_LB_TXPROD; |
3938 | adapter->rx_prod_offset = VMXNET3_REG_LB_RXPROD; |
3939 | adapter->rx_prod2_offset = VMXNET3_REG_LB_RXPROD2; |
3940 | } else { |
3941 | adapter->tx_prod_offset = VMXNET3_REG_TXPROD; |
3942 | adapter->rx_prod_offset = VMXNET3_REG_RXPROD; |
3943 | adapter->rx_prod2_offset = VMXNET3_REG_RXPROD2; |
3944 | } |
3945 | |
3946 | if (VMXNET3_VERSION_GE_6(adapter)) { |
3947 | spin_lock_irqsave(&adapter->cmd_lock, flags); |
3948 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, |
3949 | VMXNET3_CMD_GET_MAX_QUEUES_CONF); |
3950 | queues = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); |
3951 | spin_unlock_irqrestore(lock: &adapter->cmd_lock, flags); |
3952 | if (queues > 0) { |
3953 | adapter->num_rx_queues = min(num_rx_queues, ((queues >> 8) & 0xff)); |
3954 | adapter->num_tx_queues = min(num_tx_queues, (queues & 0xff)); |
3955 | } else { |
3956 | adapter->num_rx_queues = min(num_rx_queues, |
3957 | VMXNET3_DEVICE_DEFAULT_RX_QUEUES); |
3958 | adapter->num_tx_queues = min(num_tx_queues, |
3959 | VMXNET3_DEVICE_DEFAULT_TX_QUEUES); |
3960 | } |
3961 | if (adapter->num_rx_queues > VMXNET3_MAX_RX_QUEUES || |
3962 | adapter->num_tx_queues > VMXNET3_MAX_TX_QUEUES) { |
3963 | adapter->queuesExtEnabled = true; |
3964 | } else { |
3965 | adapter->queuesExtEnabled = false; |
3966 | } |
3967 | } else { |
3968 | adapter->queuesExtEnabled = false; |
3969 | num_rx_queues = rounddown_pow_of_two(num_rx_queues); |
3970 | num_tx_queues = rounddown_pow_of_two(num_tx_queues); |
3971 | adapter->num_rx_queues = min(num_rx_queues, |
3972 | VMXNET3_DEVICE_DEFAULT_RX_QUEUES); |
3973 | adapter->num_tx_queues = min(num_tx_queues, |
3974 | VMXNET3_DEVICE_DEFAULT_TX_QUEUES); |
3975 | } |
3976 | dev_info(&pdev->dev, |
3977 | "# of Tx queues : %d, # of Rx queues : %d\n" , |
3978 | adapter->num_tx_queues, adapter->num_rx_queues); |
3979 | |
3980 | adapter->rx_buf_per_pkt = 1; |
3981 | |
3982 | size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues; |
3983 | size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues; |
3984 | adapter->tqd_start = dma_alloc_coherent(dev: &adapter->pdev->dev, size, |
3985 | dma_handle: &adapter->queue_desc_pa, |
3986 | GFP_KERNEL); |
3987 | |
3988 | if (!adapter->tqd_start) { |
3989 | dev_err(&pdev->dev, "Failed to allocate memory\n" ); |
3990 | err = -ENOMEM; |
3991 | goto err_ver; |
3992 | } |
3993 | adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start + |
3994 | adapter->num_tx_queues); |
3995 | |
3996 | adapter->pm_conf = dma_alloc_coherent(dev: &adapter->pdev->dev, |
3997 | size: sizeof(struct Vmxnet3_PMConf), |
3998 | dma_handle: &adapter->pm_conf_pa, |
3999 | GFP_KERNEL); |
4000 | if (adapter->pm_conf == NULL) { |
4001 | err = -ENOMEM; |
4002 | goto err_alloc_pm; |
4003 | } |
4004 | |
4005 | #ifdef VMXNET3_RSS |
4006 | |
4007 | adapter->rss_conf = dma_alloc_coherent(dev: &adapter->pdev->dev, |
4008 | size: sizeof(struct UPT1_RSSConf), |
4009 | dma_handle: &adapter->rss_conf_pa, |
4010 | GFP_KERNEL); |
4011 | if (adapter->rss_conf == NULL) { |
4012 | err = -ENOMEM; |
4013 | goto err_alloc_rss; |
4014 | } |
4015 | #endif /* VMXNET3_RSS */ |
4016 | |
4017 | if (VMXNET3_VERSION_GE_3(adapter)) { |
4018 | adapter->coal_conf = |
4019 | dma_alloc_coherent(dev: &adapter->pdev->dev, |
4020 | size: sizeof(struct Vmxnet3_CoalesceScheme) |
4021 | , |
4022 | dma_handle: &adapter->coal_conf_pa, |
4023 | GFP_KERNEL); |
4024 | if (!adapter->coal_conf) { |
4025 | err = -ENOMEM; |
4026 | goto err_coal_conf; |
4027 | } |
4028 | adapter->coal_conf->coalMode = VMXNET3_COALESCE_DISABLED; |
4029 | adapter->default_coal_mode = true; |
4030 | } |
4031 | |
4032 | if (VMXNET3_VERSION_GE_4(adapter)) { |
4033 | adapter->default_rss_fields = true; |
4034 | adapter->rss_fields = VMXNET3_RSS_FIELDS_DEFAULT; |
4035 | } |
4036 | |
4037 | SET_NETDEV_DEV(netdev, &pdev->dev); |
4038 | vmxnet3_declare_features(adapter); |
4039 | netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | |
4040 | NETDEV_XDP_ACT_NDO_XMIT; |
4041 | |
4042 | adapter->rxdata_desc_size = VMXNET3_VERSION_GE_3(adapter) ? |
4043 | VMXNET3_DEF_RXDATA_DESC_SIZE : 0; |
4044 | |
4045 | if (adapter->num_tx_queues == adapter->num_rx_queues) |
4046 | adapter->share_intr = VMXNET3_INTR_BUDDYSHARE; |
4047 | else |
4048 | adapter->share_intr = VMXNET3_INTR_DONTSHARE; |
4049 | |
4050 | vmxnet3_alloc_intr_resources(adapter); |
4051 | |
4052 | #ifdef VMXNET3_RSS |
4053 | if (adapter->num_rx_queues > 1 && |
4054 | adapter->intr.type == VMXNET3_IT_MSIX) { |
4055 | adapter->rss = true; |
4056 | netdev->hw_features |= NETIF_F_RXHASH; |
4057 | netdev->features |= NETIF_F_RXHASH; |
4058 | dev_dbg(&pdev->dev, "RSS is enabled.\n" ); |
4059 | } else { |
4060 | adapter->rss = false; |
4061 | } |
4062 | #endif |
4063 | |
4064 | vmxnet3_read_mac_addr(adapter, mac); |
4065 | dev_addr_set(dev: netdev, addr: mac); |
4066 | |
4067 | netdev->netdev_ops = &vmxnet3_netdev_ops; |
4068 | vmxnet3_set_ethtool_ops(netdev); |
4069 | netdev->watchdog_timeo = 5 * HZ; |
4070 | |
4071 | /* MTU range: 60 - 9190 */ |
4072 | netdev->min_mtu = VMXNET3_MIN_MTU; |
4073 | if (VMXNET3_VERSION_GE_6(adapter)) |
4074 | netdev->max_mtu = VMXNET3_V6_MAX_MTU; |
4075 | else |
4076 | netdev->max_mtu = VMXNET3_MAX_MTU; |
4077 | |
4078 | INIT_WORK(&adapter->work, vmxnet3_reset_work); |
4079 | set_bit(VMXNET3_STATE_BIT_QUIESCED, addr: &adapter->state); |
4080 | |
4081 | if (adapter->intr.type == VMXNET3_IT_MSIX) { |
4082 | int i; |
4083 | for (i = 0; i < adapter->num_rx_queues; i++) { |
4084 | netif_napi_add(dev: adapter->netdev, |
4085 | napi: &adapter->rx_queue[i].napi, |
4086 | poll: vmxnet3_poll_rx_only); |
4087 | } |
4088 | } else { |
4089 | netif_napi_add(dev: adapter->netdev, napi: &adapter->rx_queue[0].napi, |
4090 | poll: vmxnet3_poll); |
4091 | } |
4092 | |
4093 | netif_set_real_num_tx_queues(dev: adapter->netdev, txq: adapter->num_tx_queues); |
4094 | netif_set_real_num_rx_queues(dev: adapter->netdev, rxq: adapter->num_rx_queues); |
4095 | |
4096 | netif_carrier_off(dev: netdev); |
4097 | err = register_netdev(dev: netdev); |
4098 | |
4099 | if (err) { |
4100 | dev_err(&pdev->dev, "Failed to register adapter\n" ); |
4101 | goto err_register; |
4102 | } |
4103 | |
4104 | vmxnet3_check_link(adapter, affectTxQueue: false); |
4105 | return 0; |
4106 | |
4107 | err_register: |
4108 | if (VMXNET3_VERSION_GE_3(adapter)) { |
4109 | dma_free_coherent(dev: &adapter->pdev->dev, |
4110 | size: sizeof(struct Vmxnet3_CoalesceScheme), |
4111 | cpu_addr: adapter->coal_conf, dma_handle: adapter->coal_conf_pa); |
4112 | } |
4113 | vmxnet3_free_intr_resources(adapter); |
4114 | err_coal_conf: |
4115 | #ifdef VMXNET3_RSS |
4116 | dma_free_coherent(dev: &adapter->pdev->dev, size: sizeof(struct UPT1_RSSConf), |
4117 | cpu_addr: adapter->rss_conf, dma_handle: adapter->rss_conf_pa); |
4118 | : |
4119 | #endif |
4120 | dma_free_coherent(dev: &adapter->pdev->dev, size: sizeof(struct Vmxnet3_PMConf), |
4121 | cpu_addr: adapter->pm_conf, dma_handle: adapter->pm_conf_pa); |
4122 | err_alloc_pm: |
4123 | dma_free_coherent(dev: &adapter->pdev->dev, size, cpu_addr: adapter->tqd_start, |
4124 | dma_handle: adapter->queue_desc_pa); |
4125 | err_ver: |
4126 | vmxnet3_free_pci_resources(adapter); |
4127 | err_alloc_pci: |
4128 | dma_free_coherent(dev: &adapter->pdev->dev, |
4129 | size: sizeof(struct Vmxnet3_DriverShared), |
4130 | cpu_addr: adapter->shared, dma_handle: adapter->shared_pa); |
4131 | err_alloc_shared: |
4132 | dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa, |
4133 | sizeof(struct vmxnet3_adapter), DMA_TO_DEVICE); |
4134 | err_set_mask: |
4135 | free_netdev(dev: netdev); |
4136 | return err; |
4137 | } |
4138 | |
4139 | |
4140 | static void |
4141 | vmxnet3_remove_device(struct pci_dev *pdev) |
4142 | { |
4143 | struct net_device *netdev = pci_get_drvdata(pdev); |
4144 | struct vmxnet3_adapter *adapter = netdev_priv(dev: netdev); |
4145 | int size = 0; |
4146 | int num_rx_queues, rx_queues; |
4147 | unsigned long flags; |
4148 | |
4149 | #ifdef VMXNET3_RSS |
4150 | if (enable_mq) |
4151 | num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES, |
4152 | (int)num_online_cpus()); |
4153 | else |
4154 | #endif |
4155 | num_rx_queues = 1; |
4156 | if (!VMXNET3_VERSION_GE_6(adapter)) { |
4157 | num_rx_queues = rounddown_pow_of_two(num_rx_queues); |
4158 | } |
4159 | if (VMXNET3_VERSION_GE_6(adapter)) { |
4160 | spin_lock_irqsave(&adapter->cmd_lock, flags); |
4161 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, |
4162 | VMXNET3_CMD_GET_MAX_QUEUES_CONF); |
4163 | rx_queues = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); |
4164 | spin_unlock_irqrestore(lock: &adapter->cmd_lock, flags); |
4165 | if (rx_queues > 0) |
4166 | rx_queues = (rx_queues >> 8) & 0xff; |
4167 | else |
4168 | rx_queues = min(num_rx_queues, VMXNET3_DEVICE_DEFAULT_RX_QUEUES); |
4169 | num_rx_queues = min(num_rx_queues, rx_queues); |
4170 | } else { |
4171 | num_rx_queues = min(num_rx_queues, |
4172 | VMXNET3_DEVICE_DEFAULT_RX_QUEUES); |
4173 | } |
4174 | |
4175 | cancel_work_sync(work: &adapter->work); |
4176 | |
4177 | unregister_netdev(dev: netdev); |
4178 | |
4179 | vmxnet3_free_intr_resources(adapter); |
4180 | vmxnet3_free_pci_resources(adapter); |
4181 | if (VMXNET3_VERSION_GE_3(adapter)) { |
4182 | dma_free_coherent(dev: &adapter->pdev->dev, |
4183 | size: sizeof(struct Vmxnet3_CoalesceScheme), |
4184 | cpu_addr: adapter->coal_conf, dma_handle: adapter->coal_conf_pa); |
4185 | } |
4186 | #ifdef VMXNET3_RSS |
4187 | dma_free_coherent(dev: &adapter->pdev->dev, size: sizeof(struct UPT1_RSSConf), |
4188 | cpu_addr: adapter->rss_conf, dma_handle: adapter->rss_conf_pa); |
4189 | #endif |
4190 | dma_free_coherent(dev: &adapter->pdev->dev, size: sizeof(struct Vmxnet3_PMConf), |
4191 | cpu_addr: adapter->pm_conf, dma_handle: adapter->pm_conf_pa); |
4192 | |
4193 | size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues; |
4194 | size += sizeof(struct Vmxnet3_RxQueueDesc) * num_rx_queues; |
4195 | dma_free_coherent(dev: &adapter->pdev->dev, size, cpu_addr: adapter->tqd_start, |
4196 | dma_handle: adapter->queue_desc_pa); |
4197 | dma_free_coherent(dev: &adapter->pdev->dev, |
4198 | size: sizeof(struct Vmxnet3_DriverShared), |
4199 | cpu_addr: adapter->shared, dma_handle: adapter->shared_pa); |
4200 | dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa, |
4201 | sizeof(struct vmxnet3_adapter), DMA_TO_DEVICE); |
4202 | free_netdev(dev: netdev); |
4203 | } |
4204 | |
4205 | static void vmxnet3_shutdown_device(struct pci_dev *pdev) |
4206 | { |
4207 | struct net_device *netdev = pci_get_drvdata(pdev); |
4208 | struct vmxnet3_adapter *adapter = netdev_priv(dev: netdev); |
4209 | unsigned long flags; |
4210 | |
4211 | /* Reset_work may be in the middle of resetting the device, wait for its |
4212 | * completion. |
4213 | */ |
4214 | while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, addr: &adapter->state)) |
4215 | usleep_range(min: 1000, max: 2000); |
4216 | |
4217 | if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, |
4218 | addr: &adapter->state)) { |
4219 | clear_bit(VMXNET3_STATE_BIT_RESETTING, addr: &adapter->state); |
4220 | return; |
4221 | } |
4222 | spin_lock_irqsave(&adapter->cmd_lock, flags); |
4223 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, |
4224 | VMXNET3_CMD_QUIESCE_DEV); |
4225 | spin_unlock_irqrestore(lock: &adapter->cmd_lock, flags); |
4226 | vmxnet3_disable_all_intrs(adapter); |
4227 | |
4228 | clear_bit(VMXNET3_STATE_BIT_RESETTING, addr: &adapter->state); |
4229 | } |
4230 | |
4231 | |
4232 | #ifdef CONFIG_PM |
4233 | |
4234 | static int |
4235 | vmxnet3_suspend(struct device *device) |
4236 | { |
4237 | struct pci_dev *pdev = to_pci_dev(device); |
4238 | struct net_device *netdev = pci_get_drvdata(pdev); |
4239 | struct vmxnet3_adapter *adapter = netdev_priv(dev: netdev); |
4240 | struct Vmxnet3_PMConf *pmConf; |
4241 | struct ethhdr *ehdr; |
4242 | struct arphdr *ahdr; |
4243 | u8 *arpreq; |
4244 | struct in_device *in_dev; |
4245 | struct in_ifaddr *ifa; |
4246 | unsigned long flags; |
4247 | int i = 0; |
4248 | |
4249 | if (!netif_running(dev: netdev)) |
4250 | return 0; |
4251 | |
4252 | for (i = 0; i < adapter->num_rx_queues; i++) |
4253 | napi_disable(n: &adapter->rx_queue[i].napi); |
4254 | |
4255 | vmxnet3_disable_all_intrs(adapter); |
4256 | vmxnet3_free_irqs(adapter); |
4257 | vmxnet3_free_intr_resources(adapter); |
4258 | |
4259 | netif_device_detach(dev: netdev); |
4260 | |
4261 | /* Create wake-up filters. */ |
4262 | pmConf = adapter->pm_conf; |
4263 | memset(pmConf, 0, sizeof(*pmConf)); |
4264 | |
4265 | if (adapter->wol & WAKE_UCAST) { |
4266 | pmConf->filters[i].patternSize = ETH_ALEN; |
4267 | pmConf->filters[i].maskSize = 1; |
4268 | memcpy(pmConf->filters[i].pattern, netdev->dev_addr, ETH_ALEN); |
4269 | pmConf->filters[i].mask[0] = 0x3F; /* LSB ETH_ALEN bits */ |
4270 | |
4271 | pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER; |
4272 | i++; |
4273 | } |
4274 | |
4275 | if (adapter->wol & WAKE_ARP) { |
4276 | rcu_read_lock(); |
4277 | |
4278 | in_dev = __in_dev_get_rcu(dev: netdev); |
4279 | if (!in_dev) { |
4280 | rcu_read_unlock(); |
4281 | goto skip_arp; |
4282 | } |
4283 | |
4284 | ifa = rcu_dereference(in_dev->ifa_list); |
4285 | if (!ifa) { |
4286 | rcu_read_unlock(); |
4287 | goto skip_arp; |
4288 | } |
4289 | |
4290 | pmConf->filters[i].patternSize = ETH_HLEN + /* Ethernet header*/ |
4291 | sizeof(struct arphdr) + /* ARP header */ |
4292 | 2 * ETH_ALEN + /* 2 Ethernet addresses*/ |
4293 | 2 * sizeof(u32); /*2 IPv4 addresses */ |
4294 | pmConf->filters[i].maskSize = |
4295 | (pmConf->filters[i].patternSize - 1) / 8 + 1; |
4296 | |
4297 | /* ETH_P_ARP in Ethernet header. */ |
4298 | ehdr = (struct ethhdr *)pmConf->filters[i].pattern; |
4299 | ehdr->h_proto = htons(ETH_P_ARP); |
4300 | |
4301 | /* ARPOP_REQUEST in ARP header. */ |
4302 | ahdr = (struct arphdr *)&pmConf->filters[i].pattern[ETH_HLEN]; |
4303 | ahdr->ar_op = htons(ARPOP_REQUEST); |
4304 | arpreq = (u8 *)(ahdr + 1); |
4305 | |
4306 | /* The Unicast IPv4 address in 'tip' field. */ |
4307 | arpreq += 2 * ETH_ALEN + sizeof(u32); |
4308 | *(__be32 *)arpreq = ifa->ifa_address; |
4309 | |
4310 | rcu_read_unlock(); |
4311 | |
4312 | /* The mask for the relevant bits. */ |
4313 | pmConf->filters[i].mask[0] = 0x00; |
4314 | pmConf->filters[i].mask[1] = 0x30; /* ETH_P_ARP */ |
4315 | pmConf->filters[i].mask[2] = 0x30; /* ARPOP_REQUEST */ |
4316 | pmConf->filters[i].mask[3] = 0x00; |
4317 | pmConf->filters[i].mask[4] = 0xC0; /* IPv4 TIP */ |
4318 | pmConf->filters[i].mask[5] = 0x03; /* IPv4 TIP */ |
4319 | |
4320 | pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER; |
4321 | i++; |
4322 | } |
4323 | |
4324 | skip_arp: |
4325 | if (adapter->wol & WAKE_MAGIC) |
4326 | pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_MAGIC; |
4327 | |
4328 | pmConf->numFilters = i; |
4329 | |
4330 | adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1); |
4331 | adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof( |
4332 | *pmConf)); |
4333 | adapter->shared->devRead.pmConfDesc.confPA = |
4334 | cpu_to_le64(adapter->pm_conf_pa); |
4335 | |
4336 | spin_lock_irqsave(&adapter->cmd_lock, flags); |
4337 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, |
4338 | VMXNET3_CMD_UPDATE_PMCFG); |
4339 | spin_unlock_irqrestore(lock: &adapter->cmd_lock, flags); |
4340 | |
4341 | pci_save_state(dev: pdev); |
4342 | pci_enable_wake(dev: pdev, state: pci_choose_state(dev: pdev, PMSG_SUSPEND), |
4343 | enable: adapter->wol); |
4344 | pci_disable_device(dev: pdev); |
4345 | pci_set_power_state(dev: pdev, state: pci_choose_state(dev: pdev, PMSG_SUSPEND)); |
4346 | |
4347 | return 0; |
4348 | } |
4349 | |
4350 | |
4351 | static int |
4352 | vmxnet3_resume(struct device *device) |
4353 | { |
4354 | int err; |
4355 | unsigned long flags; |
4356 | struct pci_dev *pdev = to_pci_dev(device); |
4357 | struct net_device *netdev = pci_get_drvdata(pdev); |
4358 | struct vmxnet3_adapter *adapter = netdev_priv(dev: netdev); |
4359 | |
4360 | if (!netif_running(dev: netdev)) |
4361 | return 0; |
4362 | |
4363 | pci_set_power_state(dev: pdev, PCI_D0); |
4364 | pci_restore_state(dev: pdev); |
4365 | err = pci_enable_device_mem(dev: pdev); |
4366 | if (err != 0) |
4367 | return err; |
4368 | |
4369 | pci_enable_wake(dev: pdev, PCI_D0, enable: 0); |
4370 | |
4371 | vmxnet3_alloc_intr_resources(adapter); |
4372 | |
4373 | /* During hibernate and suspend, device has to be reinitialized as the |
4374 | * device state need not be preserved. |
4375 | */ |
4376 | |
4377 | /* Need not check adapter state as other reset tasks cannot run during |
4378 | * device resume. |
4379 | */ |
4380 | spin_lock_irqsave(&adapter->cmd_lock, flags); |
4381 | VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, |
4382 | VMXNET3_CMD_QUIESCE_DEV); |
4383 | spin_unlock_irqrestore(lock: &adapter->cmd_lock, flags); |
4384 | vmxnet3_tq_cleanup_all(adapter); |
4385 | vmxnet3_rq_cleanup_all(adapter); |
4386 | |
4387 | vmxnet3_reset_dev(adapter); |
4388 | err = vmxnet3_activate_dev(adapter); |
4389 | if (err != 0) { |
4390 | netdev_err(dev: netdev, |
4391 | format: "failed to re-activate on resume, error: %d" , err); |
4392 | vmxnet3_force_close(adapter); |
4393 | return err; |
4394 | } |
4395 | netif_device_attach(dev: netdev); |
4396 | |
4397 | return 0; |
4398 | } |
4399 | |
4400 | static const struct dev_pm_ops vmxnet3_pm_ops = { |
4401 | .suspend = vmxnet3_suspend, |
4402 | .resume = vmxnet3_resume, |
4403 | .freeze = vmxnet3_suspend, |
4404 | .restore = vmxnet3_resume, |
4405 | }; |
4406 | #endif |
4407 | |
4408 | static struct pci_driver vmxnet3_driver = { |
4409 | .name = vmxnet3_driver_name, |
4410 | .id_table = vmxnet3_pciid_table, |
4411 | .probe = vmxnet3_probe_device, |
4412 | .remove = vmxnet3_remove_device, |
4413 | .shutdown = vmxnet3_shutdown_device, |
4414 | #ifdef CONFIG_PM |
4415 | .driver.pm = &vmxnet3_pm_ops, |
4416 | #endif |
4417 | }; |
4418 | |
4419 | |
4420 | static int __init |
4421 | vmxnet3_init_module(void) |
4422 | { |
4423 | pr_info("%s - version %s\n" , VMXNET3_DRIVER_DESC, |
4424 | VMXNET3_DRIVER_VERSION_REPORT); |
4425 | return pci_register_driver(&vmxnet3_driver); |
4426 | } |
4427 | |
4428 | module_init(vmxnet3_init_module); |
4429 | |
4430 | |
4431 | static void |
4432 | vmxnet3_exit_module(void) |
4433 | { |
4434 | pci_unregister_driver(dev: &vmxnet3_driver); |
4435 | } |
4436 | |
4437 | module_exit(vmxnet3_exit_module); |
4438 | |
4439 | MODULE_AUTHOR("VMware, Inc." ); |
4440 | MODULE_DESCRIPTION(VMXNET3_DRIVER_DESC); |
4441 | MODULE_LICENSE("GPL v2" ); |
4442 | MODULE_VERSION(VMXNET3_DRIVER_VERSION_STRING); |
4443 | |