1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /**************************************************************************** |
3 | * Driver for Solarflare network controllers and boards |
4 | * Copyright 2005-2006 Fen Systems Ltd. |
5 | * Copyright 2005-2013 Solarflare Communications Inc. |
6 | */ |
7 | |
8 | #include <linux/pci.h> |
9 | #include <linux/tcp.h> |
10 | #include <linux/ip.h> |
11 | #include <linux/in.h> |
12 | #include <linux/ipv6.h> |
13 | #include <linux/slab.h> |
14 | #include <net/ipv6.h> |
15 | #include <linux/if_ether.h> |
16 | #include <linux/highmem.h> |
17 | #include <linux/cache.h> |
18 | #include "net_driver.h" |
19 | #include "efx.h" |
20 | #include "io.h" |
21 | #include "nic.h" |
22 | #include "tx.h" |
23 | #include "workarounds.h" |
24 | |
25 | static inline u8 *ef4_tx_get_copy_buffer(struct ef4_tx_queue *tx_queue, |
26 | struct ef4_tx_buffer *buffer) |
27 | { |
28 | unsigned int index = ef4_tx_queue_get_insert_index(tx_queue); |
29 | struct ef4_buffer *page_buf = |
30 | &tx_queue->cb_page[index >> (PAGE_SHIFT - EF4_TX_CB_ORDER)]; |
31 | unsigned int offset = |
32 | ((index << EF4_TX_CB_ORDER) + NET_IP_ALIGN) & (PAGE_SIZE - 1); |
33 | |
34 | if (unlikely(!page_buf->addr) && |
35 | ef4_nic_alloc_buffer(efx: tx_queue->efx, buffer: page_buf, PAGE_SIZE, |
36 | GFP_ATOMIC)) |
37 | return NULL; |
38 | buffer->dma_addr = page_buf->dma_addr + offset; |
39 | buffer->unmap_len = 0; |
40 | return (u8 *)page_buf->addr + offset; |
41 | } |
42 | |
43 | u8 *ef4_tx_get_copy_buffer_limited(struct ef4_tx_queue *tx_queue, |
44 | struct ef4_tx_buffer *buffer, size_t len) |
45 | { |
46 | if (len > EF4_TX_CB_SIZE) |
47 | return NULL; |
48 | return ef4_tx_get_copy_buffer(tx_queue, buffer); |
49 | } |
50 | |
51 | static void ef4_dequeue_buffer(struct ef4_tx_queue *tx_queue, |
52 | struct ef4_tx_buffer *buffer, |
53 | unsigned int *pkts_compl, |
54 | unsigned int *bytes_compl) |
55 | { |
56 | if (buffer->unmap_len) { |
57 | struct device *dma_dev = &tx_queue->efx->pci_dev->dev; |
58 | dma_addr_t unmap_addr = buffer->dma_addr - buffer->dma_offset; |
59 | if (buffer->flags & EF4_TX_BUF_MAP_SINGLE) |
60 | dma_unmap_single(dma_dev, unmap_addr, buffer->unmap_len, |
61 | DMA_TO_DEVICE); |
62 | else |
63 | dma_unmap_page(dma_dev, unmap_addr, buffer->unmap_len, |
64 | DMA_TO_DEVICE); |
65 | buffer->unmap_len = 0; |
66 | } |
67 | |
68 | if (buffer->flags & EF4_TX_BUF_SKB) { |
69 | (*pkts_compl)++; |
70 | (*bytes_compl) += buffer->skb->len; |
71 | dev_consume_skb_any(skb: (struct sk_buff *)buffer->skb); |
72 | netif_vdbg(tx_queue->efx, tx_done, tx_queue->efx->net_dev, |
73 | "TX queue %d transmission id %x complete\n" , |
74 | tx_queue->queue, tx_queue->read_count); |
75 | } |
76 | |
77 | buffer->len = 0; |
78 | buffer->flags = 0; |
79 | } |
80 | |
81 | unsigned int ef4_tx_max_skb_descs(struct ef4_nic *efx) |
82 | { |
83 | /* This is probably too much since we don't have any TSO support; |
84 | * it's a left-over from when we had Software TSO. But it's safer |
85 | * to leave it as-is than try to determine a new bound. |
86 | */ |
87 | /* Header and payload descriptor for each output segment, plus |
88 | * one for every input fragment boundary within a segment |
89 | */ |
90 | unsigned int max_descs = EF4_TSO_MAX_SEGS * 2 + MAX_SKB_FRAGS; |
91 | |
92 | /* Possibly one more per segment for the alignment workaround, |
93 | * or for option descriptors |
94 | */ |
95 | if (EF4_WORKAROUND_5391(efx)) |
96 | max_descs += EF4_TSO_MAX_SEGS; |
97 | |
98 | /* Possibly more for PCIe page boundaries within input fragments */ |
99 | if (PAGE_SIZE > EF4_PAGE_SIZE) |
100 | max_descs += max_t(unsigned int, MAX_SKB_FRAGS, |
101 | DIV_ROUND_UP(GSO_LEGACY_MAX_SIZE, |
102 | EF4_PAGE_SIZE)); |
103 | |
104 | return max_descs; |
105 | } |
106 | |
107 | static void ef4_tx_maybe_stop_queue(struct ef4_tx_queue *txq1) |
108 | { |
109 | /* We need to consider both queues that the net core sees as one */ |
110 | struct ef4_tx_queue *txq2 = ef4_tx_queue_partner(tx_queue: txq1); |
111 | struct ef4_nic *efx = txq1->efx; |
112 | unsigned int fill_level; |
113 | |
114 | fill_level = max(txq1->insert_count - txq1->old_read_count, |
115 | txq2->insert_count - txq2->old_read_count); |
116 | if (likely(fill_level < efx->txq_stop_thresh)) |
117 | return; |
118 | |
119 | /* We used the stale old_read_count above, which gives us a |
120 | * pessimistic estimate of the fill level (which may even |
121 | * validly be >= efx->txq_entries). Now try again using |
122 | * read_count (more likely to be a cache miss). |
123 | * |
124 | * If we read read_count and then conditionally stop the |
125 | * queue, it is possible for the completion path to race with |
126 | * us and complete all outstanding descriptors in the middle, |
127 | * after which there will be no more completions to wake it. |
128 | * Therefore we stop the queue first, then read read_count |
129 | * (with a memory barrier to ensure the ordering), then |
130 | * restart the queue if the fill level turns out to be low |
131 | * enough. |
132 | */ |
133 | netif_tx_stop_queue(dev_queue: txq1->core_txq); |
134 | smp_mb(); |
135 | txq1->old_read_count = READ_ONCE(txq1->read_count); |
136 | txq2->old_read_count = READ_ONCE(txq2->read_count); |
137 | |
138 | fill_level = max(txq1->insert_count - txq1->old_read_count, |
139 | txq2->insert_count - txq2->old_read_count); |
140 | EF4_BUG_ON_PARANOID(fill_level >= efx->txq_entries); |
141 | if (likely(fill_level < efx->txq_stop_thresh)) { |
142 | smp_mb(); |
143 | if (likely(!efx->loopback_selftest)) |
144 | netif_tx_start_queue(dev_queue: txq1->core_txq); |
145 | } |
146 | } |
147 | |
148 | static int ef4_enqueue_skb_copy(struct ef4_tx_queue *tx_queue, |
149 | struct sk_buff *skb) |
150 | { |
151 | unsigned int min_len = tx_queue->tx_min_size; |
152 | unsigned int copy_len = skb->len; |
153 | struct ef4_tx_buffer *buffer; |
154 | u8 *copy_buffer; |
155 | int rc; |
156 | |
157 | EF4_BUG_ON_PARANOID(copy_len > EF4_TX_CB_SIZE); |
158 | |
159 | buffer = ef4_tx_queue_get_insert_buffer(tx_queue); |
160 | |
161 | copy_buffer = ef4_tx_get_copy_buffer(tx_queue, buffer); |
162 | if (unlikely(!copy_buffer)) |
163 | return -ENOMEM; |
164 | |
165 | rc = skb_copy_bits(skb, offset: 0, to: copy_buffer, len: copy_len); |
166 | EF4_WARN_ON_PARANOID(rc); |
167 | if (unlikely(copy_len < min_len)) { |
168 | memset(copy_buffer + copy_len, 0, min_len - copy_len); |
169 | buffer->len = min_len; |
170 | } else { |
171 | buffer->len = copy_len; |
172 | } |
173 | |
174 | buffer->skb = skb; |
175 | buffer->flags = EF4_TX_BUF_SKB; |
176 | |
177 | ++tx_queue->insert_count; |
178 | return rc; |
179 | } |
180 | |
181 | static struct ef4_tx_buffer *ef4_tx_map_chunk(struct ef4_tx_queue *tx_queue, |
182 | dma_addr_t dma_addr, |
183 | size_t len) |
184 | { |
185 | const struct ef4_nic_type *nic_type = tx_queue->efx->type; |
186 | struct ef4_tx_buffer *buffer; |
187 | unsigned int dma_len; |
188 | |
189 | /* Map the fragment taking account of NIC-dependent DMA limits. */ |
190 | do { |
191 | buffer = ef4_tx_queue_get_insert_buffer(tx_queue); |
192 | dma_len = nic_type->tx_limit_len(tx_queue, dma_addr, len); |
193 | |
194 | buffer->len = dma_len; |
195 | buffer->dma_addr = dma_addr; |
196 | buffer->flags = EF4_TX_BUF_CONT; |
197 | len -= dma_len; |
198 | dma_addr += dma_len; |
199 | ++tx_queue->insert_count; |
200 | } while (len); |
201 | |
202 | return buffer; |
203 | } |
204 | |
205 | /* Map all data from an SKB for DMA and create descriptors on the queue. |
206 | */ |
207 | static int ef4_tx_map_data(struct ef4_tx_queue *tx_queue, struct sk_buff *skb) |
208 | { |
209 | struct ef4_nic *efx = tx_queue->efx; |
210 | struct device *dma_dev = &efx->pci_dev->dev; |
211 | unsigned int frag_index, nr_frags; |
212 | dma_addr_t dma_addr, unmap_addr; |
213 | unsigned short dma_flags; |
214 | size_t len, unmap_len; |
215 | |
216 | nr_frags = skb_shinfo(skb)->nr_frags; |
217 | frag_index = 0; |
218 | |
219 | /* Map header data. */ |
220 | len = skb_headlen(skb); |
221 | dma_addr = dma_map_single(dma_dev, skb->data, len, DMA_TO_DEVICE); |
222 | dma_flags = EF4_TX_BUF_MAP_SINGLE; |
223 | unmap_len = len; |
224 | unmap_addr = dma_addr; |
225 | |
226 | if (unlikely(dma_mapping_error(dma_dev, dma_addr))) |
227 | return -EIO; |
228 | |
229 | /* Add descriptors for each fragment. */ |
230 | do { |
231 | struct ef4_tx_buffer *buffer; |
232 | skb_frag_t *fragment; |
233 | |
234 | buffer = ef4_tx_map_chunk(tx_queue, dma_addr, len); |
235 | |
236 | /* The final descriptor for a fragment is responsible for |
237 | * unmapping the whole fragment. |
238 | */ |
239 | buffer->flags = EF4_TX_BUF_CONT | dma_flags; |
240 | buffer->unmap_len = unmap_len; |
241 | buffer->dma_offset = buffer->dma_addr - unmap_addr; |
242 | |
243 | if (frag_index >= nr_frags) { |
244 | /* Store SKB details with the final buffer for |
245 | * the completion. |
246 | */ |
247 | buffer->skb = skb; |
248 | buffer->flags = EF4_TX_BUF_SKB | dma_flags; |
249 | return 0; |
250 | } |
251 | |
252 | /* Move on to the next fragment. */ |
253 | fragment = &skb_shinfo(skb)->frags[frag_index++]; |
254 | len = skb_frag_size(frag: fragment); |
255 | dma_addr = skb_frag_dma_map(dev: dma_dev, frag: fragment, |
256 | offset: 0, size: len, dir: DMA_TO_DEVICE); |
257 | dma_flags = 0; |
258 | unmap_len = len; |
259 | unmap_addr = dma_addr; |
260 | |
261 | if (unlikely(dma_mapping_error(dma_dev, dma_addr))) |
262 | return -EIO; |
263 | } while (1); |
264 | } |
265 | |
266 | /* Remove buffers put into a tx_queue. None of the buffers must have |
267 | * an skb attached. |
268 | */ |
269 | static void ef4_enqueue_unwind(struct ef4_tx_queue *tx_queue) |
270 | { |
271 | struct ef4_tx_buffer *buffer; |
272 | |
273 | /* Work backwards until we hit the original insert pointer value */ |
274 | while (tx_queue->insert_count != tx_queue->write_count) { |
275 | --tx_queue->insert_count; |
276 | buffer = __ef4_tx_queue_get_insert_buffer(tx_queue); |
277 | ef4_dequeue_buffer(tx_queue, buffer, NULL, NULL); |
278 | } |
279 | } |
280 | |
281 | /* |
282 | * Add a socket buffer to a TX queue |
283 | * |
284 | * This maps all fragments of a socket buffer for DMA and adds them to |
285 | * the TX queue. The queue's insert pointer will be incremented by |
286 | * the number of fragments in the socket buffer. |
287 | * |
288 | * If any DMA mapping fails, any mapped fragments will be unmapped, |
289 | * the queue's insert pointer will be restored to its original value. |
290 | * |
291 | * This function is split out from ef4_hard_start_xmit to allow the |
292 | * loopback test to direct packets via specific TX queues. |
293 | * |
294 | * Returns NETDEV_TX_OK. |
295 | * You must hold netif_tx_lock() to call this function. |
296 | */ |
297 | netdev_tx_t ef4_enqueue_skb(struct ef4_tx_queue *tx_queue, struct sk_buff *skb) |
298 | { |
299 | bool data_mapped = false; |
300 | unsigned int skb_len; |
301 | |
302 | skb_len = skb->len; |
303 | EF4_WARN_ON_PARANOID(skb_is_gso(skb)); |
304 | |
305 | if (skb_len < tx_queue->tx_min_size || |
306 | (skb->data_len && skb_len <= EF4_TX_CB_SIZE)) { |
307 | /* Pad short packets or coalesce short fragmented packets. */ |
308 | if (ef4_enqueue_skb_copy(tx_queue, skb)) |
309 | goto err; |
310 | tx_queue->cb_packets++; |
311 | data_mapped = true; |
312 | } |
313 | |
314 | /* Map for DMA and create descriptors if we haven't done so already. */ |
315 | if (!data_mapped && (ef4_tx_map_data(tx_queue, skb))) |
316 | goto err; |
317 | |
318 | /* Update BQL */ |
319 | netdev_tx_sent_queue(dev_queue: tx_queue->core_txq, bytes: skb_len); |
320 | |
321 | /* Pass off to hardware */ |
322 | if (!netdev_xmit_more() || netif_xmit_stopped(dev_queue: tx_queue->core_txq)) { |
323 | struct ef4_tx_queue *txq2 = ef4_tx_queue_partner(tx_queue); |
324 | |
325 | /* There could be packets left on the partner queue if those |
326 | * SKBs had skb->xmit_more set. If we do not push those they |
327 | * could be left for a long time and cause a netdev watchdog. |
328 | */ |
329 | if (txq2->xmit_more_available) |
330 | ef4_nic_push_buffers(tx_queue: txq2); |
331 | |
332 | ef4_nic_push_buffers(tx_queue); |
333 | } else { |
334 | tx_queue->xmit_more_available = netdev_xmit_more(); |
335 | } |
336 | |
337 | tx_queue->tx_packets++; |
338 | |
339 | ef4_tx_maybe_stop_queue(txq1: tx_queue); |
340 | |
341 | return NETDEV_TX_OK; |
342 | |
343 | |
344 | err: |
345 | ef4_enqueue_unwind(tx_queue); |
346 | dev_kfree_skb_any(skb); |
347 | return NETDEV_TX_OK; |
348 | } |
349 | |
350 | /* Remove packets from the TX queue |
351 | * |
352 | * This removes packets from the TX queue, up to and including the |
353 | * specified index. |
354 | */ |
355 | static void ef4_dequeue_buffers(struct ef4_tx_queue *tx_queue, |
356 | unsigned int index, |
357 | unsigned int *pkts_compl, |
358 | unsigned int *bytes_compl) |
359 | { |
360 | struct ef4_nic *efx = tx_queue->efx; |
361 | unsigned int stop_index, read_ptr; |
362 | |
363 | stop_index = (index + 1) & tx_queue->ptr_mask; |
364 | read_ptr = tx_queue->read_count & tx_queue->ptr_mask; |
365 | |
366 | while (read_ptr != stop_index) { |
367 | struct ef4_tx_buffer *buffer = &tx_queue->buffer[read_ptr]; |
368 | |
369 | if (!(buffer->flags & EF4_TX_BUF_OPTION) && |
370 | unlikely(buffer->len == 0)) { |
371 | netif_err(efx, tx_err, efx->net_dev, |
372 | "TX queue %d spurious TX completion id %x\n" , |
373 | tx_queue->queue, read_ptr); |
374 | ef4_schedule_reset(efx, type: RESET_TYPE_TX_SKIP); |
375 | return; |
376 | } |
377 | |
378 | ef4_dequeue_buffer(tx_queue, buffer, pkts_compl, bytes_compl); |
379 | |
380 | ++tx_queue->read_count; |
381 | read_ptr = tx_queue->read_count & tx_queue->ptr_mask; |
382 | } |
383 | } |
384 | |
385 | /* Initiate a packet transmission. We use one channel per CPU |
386 | * (sharing when we have more CPUs than channels). On Falcon, the TX |
387 | * completion events will be directed back to the CPU that transmitted |
388 | * the packet, which should be cache-efficient. |
389 | * |
390 | * Context: non-blocking. |
391 | * Note that returning anything other than NETDEV_TX_OK will cause the |
392 | * OS to free the skb. |
393 | */ |
394 | netdev_tx_t ef4_hard_start_xmit(struct sk_buff *skb, |
395 | struct net_device *net_dev) |
396 | { |
397 | struct ef4_nic *efx = netdev_priv(dev: net_dev); |
398 | struct ef4_tx_queue *tx_queue; |
399 | unsigned index, type; |
400 | |
401 | EF4_WARN_ON_PARANOID(!netif_device_present(net_dev)); |
402 | |
403 | index = skb_get_queue_mapping(skb); |
404 | type = skb->ip_summed == CHECKSUM_PARTIAL ? EF4_TXQ_TYPE_OFFLOAD : 0; |
405 | if (index >= efx->n_tx_channels) { |
406 | index -= efx->n_tx_channels; |
407 | type |= EF4_TXQ_TYPE_HIGHPRI; |
408 | } |
409 | tx_queue = ef4_get_tx_queue(efx, index, type); |
410 | |
411 | return ef4_enqueue_skb(tx_queue, skb); |
412 | } |
413 | |
414 | void ef4_init_tx_queue_core_txq(struct ef4_tx_queue *tx_queue) |
415 | { |
416 | struct ef4_nic *efx = tx_queue->efx; |
417 | |
418 | /* Must be inverse of queue lookup in ef4_hard_start_xmit() */ |
419 | tx_queue->core_txq = |
420 | netdev_get_tx_queue(dev: efx->net_dev, |
421 | index: tx_queue->queue / EF4_TXQ_TYPES + |
422 | ((tx_queue->queue & EF4_TXQ_TYPE_HIGHPRI) ? |
423 | efx->n_tx_channels : 0)); |
424 | } |
425 | |
426 | int ef4_setup_tc(struct net_device *net_dev, enum tc_setup_type type, |
427 | void *type_data) |
428 | { |
429 | struct ef4_nic *efx = netdev_priv(dev: net_dev); |
430 | struct tc_mqprio_qopt *mqprio = type_data; |
431 | struct ef4_channel *channel; |
432 | struct ef4_tx_queue *tx_queue; |
433 | unsigned tc, num_tc; |
434 | int rc; |
435 | |
436 | if (type != TC_SETUP_QDISC_MQPRIO) |
437 | return -EOPNOTSUPP; |
438 | |
439 | num_tc = mqprio->num_tc; |
440 | |
441 | if (ef4_nic_rev(efx) < EF4_REV_FALCON_B0 || num_tc > EF4_MAX_TX_TC) |
442 | return -EINVAL; |
443 | |
444 | mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; |
445 | |
446 | if (num_tc == net_dev->num_tc) |
447 | return 0; |
448 | |
449 | for (tc = 0; tc < num_tc; tc++) { |
450 | net_dev->tc_to_txq[tc].offset = tc * efx->n_tx_channels; |
451 | net_dev->tc_to_txq[tc].count = efx->n_tx_channels; |
452 | } |
453 | |
454 | if (num_tc > net_dev->num_tc) { |
455 | /* Initialise high-priority queues as necessary */ |
456 | ef4_for_each_channel(channel, efx) { |
457 | ef4_for_each_possible_channel_tx_queue(tx_queue, |
458 | channel) { |
459 | if (!(tx_queue->queue & EF4_TXQ_TYPE_HIGHPRI)) |
460 | continue; |
461 | if (!tx_queue->buffer) { |
462 | rc = ef4_probe_tx_queue(tx_queue); |
463 | if (rc) |
464 | return rc; |
465 | } |
466 | if (!tx_queue->initialised) |
467 | ef4_init_tx_queue(tx_queue); |
468 | ef4_init_tx_queue_core_txq(tx_queue); |
469 | } |
470 | } |
471 | } else { |
472 | /* Reduce number of classes before number of queues */ |
473 | net_dev->num_tc = num_tc; |
474 | } |
475 | |
476 | rc = netif_set_real_num_tx_queues(dev: net_dev, |
477 | max_t(int, num_tc, 1) * |
478 | efx->n_tx_channels); |
479 | if (rc) |
480 | return rc; |
481 | |
482 | /* Do not destroy high-priority queues when they become |
483 | * unused. We would have to flush them first, and it is |
484 | * fairly difficult to flush a subset of TX queues. Leave |
485 | * it to ef4_fini_channels(). |
486 | */ |
487 | |
488 | net_dev->num_tc = num_tc; |
489 | return 0; |
490 | } |
491 | |
492 | void ef4_xmit_done(struct ef4_tx_queue *tx_queue, unsigned int index) |
493 | { |
494 | unsigned fill_level; |
495 | struct ef4_nic *efx = tx_queue->efx; |
496 | struct ef4_tx_queue *txq2; |
497 | unsigned int pkts_compl = 0, bytes_compl = 0; |
498 | |
499 | EF4_BUG_ON_PARANOID(index > tx_queue->ptr_mask); |
500 | |
501 | ef4_dequeue_buffers(tx_queue, index, pkts_compl: &pkts_compl, bytes_compl: &bytes_compl); |
502 | tx_queue->pkts_compl += pkts_compl; |
503 | tx_queue->bytes_compl += bytes_compl; |
504 | |
505 | if (pkts_compl > 1) |
506 | ++tx_queue->merge_events; |
507 | |
508 | /* See if we need to restart the netif queue. This memory |
509 | * barrier ensures that we write read_count (inside |
510 | * ef4_dequeue_buffers()) before reading the queue status. |
511 | */ |
512 | smp_mb(); |
513 | if (unlikely(netif_tx_queue_stopped(tx_queue->core_txq)) && |
514 | likely(efx->port_enabled) && |
515 | likely(netif_device_present(efx->net_dev))) { |
516 | txq2 = ef4_tx_queue_partner(tx_queue); |
517 | fill_level = max(tx_queue->insert_count - tx_queue->read_count, |
518 | txq2->insert_count - txq2->read_count); |
519 | if (fill_level <= efx->txq_wake_thresh) |
520 | netif_tx_wake_queue(dev_queue: tx_queue->core_txq); |
521 | } |
522 | |
523 | /* Check whether the hardware queue is now empty */ |
524 | if ((int)(tx_queue->read_count - tx_queue->old_write_count) >= 0) { |
525 | tx_queue->old_write_count = READ_ONCE(tx_queue->write_count); |
526 | if (tx_queue->read_count == tx_queue->old_write_count) { |
527 | smp_mb(); |
528 | tx_queue->empty_read_count = |
529 | tx_queue->read_count | EF4_EMPTY_COUNT_VALID; |
530 | } |
531 | } |
532 | } |
533 | |
534 | static unsigned int ef4_tx_cb_page_count(struct ef4_tx_queue *tx_queue) |
535 | { |
536 | return DIV_ROUND_UP(tx_queue->ptr_mask + 1, PAGE_SIZE >> EF4_TX_CB_ORDER); |
537 | } |
538 | |
539 | int ef4_probe_tx_queue(struct ef4_tx_queue *tx_queue) |
540 | { |
541 | struct ef4_nic *efx = tx_queue->efx; |
542 | unsigned int entries; |
543 | int rc; |
544 | |
545 | /* Create the smallest power-of-two aligned ring */ |
546 | entries = max(roundup_pow_of_two(efx->txq_entries), EF4_MIN_DMAQ_SIZE); |
547 | EF4_BUG_ON_PARANOID(entries > EF4_MAX_DMAQ_SIZE); |
548 | tx_queue->ptr_mask = entries - 1; |
549 | |
550 | netif_dbg(efx, probe, efx->net_dev, |
551 | "creating TX queue %d size %#x mask %#x\n" , |
552 | tx_queue->queue, efx->txq_entries, tx_queue->ptr_mask); |
553 | |
554 | /* Allocate software ring */ |
555 | tx_queue->buffer = kcalloc(n: entries, size: sizeof(*tx_queue->buffer), |
556 | GFP_KERNEL); |
557 | if (!tx_queue->buffer) |
558 | return -ENOMEM; |
559 | |
560 | tx_queue->cb_page = kcalloc(n: ef4_tx_cb_page_count(tx_queue), |
561 | size: sizeof(tx_queue->cb_page[0]), GFP_KERNEL); |
562 | if (!tx_queue->cb_page) { |
563 | rc = -ENOMEM; |
564 | goto fail1; |
565 | } |
566 | |
567 | /* Allocate hardware ring */ |
568 | rc = ef4_nic_probe_tx(tx_queue); |
569 | if (rc) |
570 | goto fail2; |
571 | |
572 | return 0; |
573 | |
574 | fail2: |
575 | kfree(objp: tx_queue->cb_page); |
576 | tx_queue->cb_page = NULL; |
577 | fail1: |
578 | kfree(objp: tx_queue->buffer); |
579 | tx_queue->buffer = NULL; |
580 | return rc; |
581 | } |
582 | |
583 | void ef4_init_tx_queue(struct ef4_tx_queue *tx_queue) |
584 | { |
585 | struct ef4_nic *efx = tx_queue->efx; |
586 | |
587 | netif_dbg(efx, drv, efx->net_dev, |
588 | "initialising TX queue %d\n" , tx_queue->queue); |
589 | |
590 | tx_queue->insert_count = 0; |
591 | tx_queue->write_count = 0; |
592 | tx_queue->old_write_count = 0; |
593 | tx_queue->read_count = 0; |
594 | tx_queue->old_read_count = 0; |
595 | tx_queue->empty_read_count = 0 | EF4_EMPTY_COUNT_VALID; |
596 | tx_queue->xmit_more_available = false; |
597 | |
598 | /* Some older hardware requires Tx writes larger than 32. */ |
599 | tx_queue->tx_min_size = EF4_WORKAROUND_15592(efx) ? 33 : 0; |
600 | |
601 | /* Set up TX descriptor ring */ |
602 | ef4_nic_init_tx(tx_queue); |
603 | |
604 | tx_queue->initialised = true; |
605 | } |
606 | |
607 | void ef4_fini_tx_queue(struct ef4_tx_queue *tx_queue) |
608 | { |
609 | struct ef4_tx_buffer *buffer; |
610 | |
611 | netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev, |
612 | "shutting down TX queue %d\n" , tx_queue->queue); |
613 | |
614 | if (!tx_queue->buffer) |
615 | return; |
616 | |
617 | /* Free any buffers left in the ring */ |
618 | while (tx_queue->read_count != tx_queue->write_count) { |
619 | unsigned int pkts_compl = 0, bytes_compl = 0; |
620 | buffer = &tx_queue->buffer[tx_queue->read_count & tx_queue->ptr_mask]; |
621 | ef4_dequeue_buffer(tx_queue, buffer, pkts_compl: &pkts_compl, bytes_compl: &bytes_compl); |
622 | |
623 | ++tx_queue->read_count; |
624 | } |
625 | tx_queue->xmit_more_available = false; |
626 | netdev_tx_reset_queue(q: tx_queue->core_txq); |
627 | } |
628 | |
629 | void ef4_remove_tx_queue(struct ef4_tx_queue *tx_queue) |
630 | { |
631 | int i; |
632 | |
633 | if (!tx_queue->buffer) |
634 | return; |
635 | |
636 | netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev, |
637 | "destroying TX queue %d\n" , tx_queue->queue); |
638 | ef4_nic_remove_tx(tx_queue); |
639 | |
640 | if (tx_queue->cb_page) { |
641 | for (i = 0; i < ef4_tx_cb_page_count(tx_queue); i++) |
642 | ef4_nic_free_buffer(efx: tx_queue->efx, |
643 | buffer: &tx_queue->cb_page[i]); |
644 | kfree(objp: tx_queue->cb_page); |
645 | tx_queue->cb_page = NULL; |
646 | } |
647 | |
648 | kfree(objp: tx_queue->buffer); |
649 | tx_queue->buffer = NULL; |
650 | } |
651 | |