1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Huawei HiNIC PCI Express Linux driver |
4 | * Copyright(c) 2017 Huawei Technologies Co., Ltd |
5 | */ |
6 | |
7 | #include <linux/if_vlan.h> |
8 | #include <linux/kernel.h> |
9 | #include <linux/netdevice.h> |
10 | #include <linux/u64_stats_sync.h> |
11 | #include <linux/errno.h> |
12 | #include <linux/types.h> |
13 | #include <linux/pci.h> |
14 | #include <linux/device.h> |
15 | #include <linux/dma-mapping.h> |
16 | #include <linux/slab.h> |
17 | #include <linux/interrupt.h> |
18 | #include <linux/skbuff.h> |
19 | #include <linux/smp.h> |
20 | #include <asm/byteorder.h> |
21 | #include <linux/ip.h> |
22 | #include <linux/tcp.h> |
23 | #include <linux/sctp.h> |
24 | #include <linux/ipv6.h> |
25 | #include <net/ipv6.h> |
26 | #include <net/checksum.h> |
27 | #include <net/ip6_checksum.h> |
28 | |
29 | #include "hinic_common.h" |
30 | #include "hinic_hw_if.h" |
31 | #include "hinic_hw_wqe.h" |
32 | #include "hinic_hw_wq.h" |
33 | #include "hinic_hw_qp.h" |
34 | #include "hinic_hw_dev.h" |
35 | #include "hinic_dev.h" |
36 | #include "hinic_tx.h" |
37 | |
38 | #define TX_IRQ_NO_PENDING 0 |
39 | #define TX_IRQ_NO_COALESC 0 |
40 | #define TX_IRQ_NO_LLI_TIMER 0 |
41 | #define TX_IRQ_NO_CREDIT 0 |
42 | #define TX_IRQ_NO_RESEND_TIMER 0 |
43 | |
44 | #define CI_UPDATE_NO_PENDING 0 |
45 | #define CI_UPDATE_NO_COALESC 0 |
46 | |
47 | #define HW_CONS_IDX(sq) be16_to_cpu(*(u16 *)((sq)->hw_ci_addr)) |
48 | |
49 | #define MIN_SKB_LEN 32 |
50 | |
51 | #define MAX_PAYLOAD_OFFSET 221 |
52 | #define TRANSPORT_OFFSET(l4_hdr, skb) ((u32)((l4_hdr) - (skb)->data)) |
53 | |
54 | union hinic_l3 { |
55 | struct iphdr *v4; |
56 | struct ipv6hdr *v6; |
57 | unsigned char *hdr; |
58 | }; |
59 | |
60 | union hinic_l4 { |
61 | struct tcphdr *tcp; |
62 | struct udphdr *udp; |
63 | unsigned char *hdr; |
64 | }; |
65 | |
66 | enum hinic_offload_type { |
67 | TX_OFFLOAD_TSO = BIT(0), |
68 | TX_OFFLOAD_CSUM = BIT(1), |
69 | TX_OFFLOAD_VLAN = BIT(2), |
70 | TX_OFFLOAD_INVALID = BIT(3), |
71 | }; |
72 | |
73 | /** |
74 | * hinic_txq_clean_stats - Clean the statistics of specific queue |
75 | * @txq: Logical Tx Queue |
76 | **/ |
77 | static void hinic_txq_clean_stats(struct hinic_txq *txq) |
78 | { |
79 | struct hinic_txq_stats *txq_stats = &txq->txq_stats; |
80 | |
81 | u64_stats_update_begin(syncp: &txq_stats->syncp); |
82 | txq_stats->pkts = 0; |
83 | txq_stats->bytes = 0; |
84 | txq_stats->tx_busy = 0; |
85 | txq_stats->tx_wake = 0; |
86 | txq_stats->tx_dropped = 0; |
87 | txq_stats->big_frags_pkts = 0; |
88 | u64_stats_update_end(syncp: &txq_stats->syncp); |
89 | } |
90 | |
91 | /** |
92 | * hinic_txq_get_stats - get statistics of Tx Queue |
93 | * @txq: Logical Tx Queue |
94 | * @stats: return updated stats here |
95 | **/ |
96 | void hinic_txq_get_stats(struct hinic_txq *txq, struct hinic_txq_stats *stats) |
97 | { |
98 | struct hinic_txq_stats *txq_stats = &txq->txq_stats; |
99 | unsigned int start; |
100 | |
101 | do { |
102 | start = u64_stats_fetch_begin(syncp: &txq_stats->syncp); |
103 | stats->pkts = txq_stats->pkts; |
104 | stats->bytes = txq_stats->bytes; |
105 | stats->tx_busy = txq_stats->tx_busy; |
106 | stats->tx_wake = txq_stats->tx_wake; |
107 | stats->tx_dropped = txq_stats->tx_dropped; |
108 | stats->big_frags_pkts = txq_stats->big_frags_pkts; |
109 | } while (u64_stats_fetch_retry(syncp: &txq_stats->syncp, start)); |
110 | } |
111 | |
112 | /** |
113 | * txq_stats_init - Initialize the statistics of specific queue |
114 | * @txq: Logical Tx Queue |
115 | **/ |
116 | static void txq_stats_init(struct hinic_txq *txq) |
117 | { |
118 | struct hinic_txq_stats *txq_stats = &txq->txq_stats; |
119 | |
120 | u64_stats_init(syncp: &txq_stats->syncp); |
121 | hinic_txq_clean_stats(txq); |
122 | } |
123 | |
124 | /** |
125 | * tx_map_skb - dma mapping for skb and return sges |
126 | * @nic_dev: nic device |
127 | * @skb: the skb |
128 | * @sges: returned sges |
129 | * |
130 | * Return 0 - Success, negative - Failure |
131 | **/ |
132 | static int tx_map_skb(struct hinic_dev *nic_dev, struct sk_buff *skb, |
133 | struct hinic_sge *sges) |
134 | { |
135 | struct hinic_hwdev *hwdev = nic_dev->hwdev; |
136 | struct hinic_hwif *hwif = hwdev->hwif; |
137 | struct pci_dev *pdev = hwif->pdev; |
138 | skb_frag_t *frag; |
139 | dma_addr_t dma_addr; |
140 | int i, j; |
141 | |
142 | dma_addr = dma_map_single(&pdev->dev, skb->data, skb_headlen(skb), |
143 | DMA_TO_DEVICE); |
144 | if (dma_mapping_error(dev: &pdev->dev, dma_addr)) { |
145 | dev_err(&pdev->dev, "Failed to map Tx skb data\n" ); |
146 | return -EFAULT; |
147 | } |
148 | |
149 | hinic_set_sge(sge: &sges[0], addr: dma_addr, len: skb_headlen(skb)); |
150 | |
151 | for (i = 0 ; i < skb_shinfo(skb)->nr_frags; i++) { |
152 | frag = &skb_shinfo(skb)->frags[i]; |
153 | |
154 | dma_addr = skb_frag_dma_map(dev: &pdev->dev, frag, offset: 0, |
155 | size: skb_frag_size(frag), |
156 | dir: DMA_TO_DEVICE); |
157 | if (dma_mapping_error(dev: &pdev->dev, dma_addr)) { |
158 | dev_err(&pdev->dev, "Failed to map Tx skb frag\n" ); |
159 | goto err_tx_map; |
160 | } |
161 | |
162 | hinic_set_sge(sge: &sges[i + 1], addr: dma_addr, len: skb_frag_size(frag)); |
163 | } |
164 | |
165 | return 0; |
166 | |
167 | err_tx_map: |
168 | for (j = 0; j < i; j++) |
169 | dma_unmap_page(&pdev->dev, hinic_sge_to_dma(&sges[j + 1]), |
170 | sges[j + 1].len, DMA_TO_DEVICE); |
171 | |
172 | dma_unmap_single(&pdev->dev, hinic_sge_to_dma(&sges[0]), sges[0].len, |
173 | DMA_TO_DEVICE); |
174 | return -EFAULT; |
175 | } |
176 | |
177 | /** |
178 | * tx_unmap_skb - unmap the dma address of the skb |
179 | * @nic_dev: nic device |
180 | * @skb: the skb |
181 | * @sges: the sges that are connected to the skb |
182 | **/ |
183 | static void tx_unmap_skb(struct hinic_dev *nic_dev, struct sk_buff *skb, |
184 | struct hinic_sge *sges) |
185 | { |
186 | struct hinic_hwdev *hwdev = nic_dev->hwdev; |
187 | struct hinic_hwif *hwif = hwdev->hwif; |
188 | struct pci_dev *pdev = hwif->pdev; |
189 | int i; |
190 | |
191 | for (i = 0; i < skb_shinfo(skb)->nr_frags ; i++) |
192 | dma_unmap_page(&pdev->dev, hinic_sge_to_dma(&sges[i + 1]), |
193 | sges[i + 1].len, DMA_TO_DEVICE); |
194 | |
195 | dma_unmap_single(&pdev->dev, hinic_sge_to_dma(&sges[0]), sges[0].len, |
196 | DMA_TO_DEVICE); |
197 | } |
198 | |
199 | static void get_inner_l3_l4_type(struct sk_buff *skb, union hinic_l3 *ip, |
200 | union hinic_l4 *l4, |
201 | enum hinic_offload_type offload_type, |
202 | enum hinic_l3_offload_type *l3_type, |
203 | u8 *l4_proto) |
204 | { |
205 | u8 *exthdr; |
206 | |
207 | if (ip->v4->version == 4) { |
208 | *l3_type = (offload_type == TX_OFFLOAD_CSUM) ? |
209 | IPV4_PKT_NO_CHKSUM_OFFLOAD : |
210 | IPV4_PKT_WITH_CHKSUM_OFFLOAD; |
211 | *l4_proto = ip->v4->protocol; |
212 | } else if (ip->v4->version == 6) { |
213 | *l3_type = IPV6_PKT; |
214 | exthdr = ip->hdr + sizeof(*ip->v6); |
215 | *l4_proto = ip->v6->nexthdr; |
216 | if (exthdr != l4->hdr) { |
217 | int start = exthdr - skb->data; |
218 | __be16 frag_off; |
219 | |
220 | ipv6_skip_exthdr(skb, start, nexthdrp: l4_proto, frag_offp: &frag_off); |
221 | } |
222 | } else { |
223 | *l3_type = L3TYPE_UNKNOWN; |
224 | *l4_proto = 0; |
225 | } |
226 | } |
227 | |
228 | static void get_inner_l4_info(struct sk_buff *skb, union hinic_l4 *l4, |
229 | enum hinic_offload_type offload_type, u8 l4_proto, |
230 | enum hinic_l4_offload_type *l4_offload, |
231 | u32 *l4_len, u32 *offset) |
232 | { |
233 | *l4_offload = OFFLOAD_DISABLE; |
234 | *offset = 0; |
235 | *l4_len = 0; |
236 | |
237 | switch (l4_proto) { |
238 | case IPPROTO_TCP: |
239 | *l4_offload = TCP_OFFLOAD_ENABLE; |
240 | /* doff in unit of 4B */ |
241 | *l4_len = l4->tcp->doff * 4; |
242 | *offset = *l4_len + TRANSPORT_OFFSET(l4->hdr, skb); |
243 | break; |
244 | |
245 | case IPPROTO_UDP: |
246 | *l4_offload = UDP_OFFLOAD_ENABLE; |
247 | *l4_len = sizeof(struct udphdr); |
248 | *offset = TRANSPORT_OFFSET(l4->hdr, skb); |
249 | break; |
250 | |
251 | case IPPROTO_SCTP: |
252 | /* only csum offload support sctp */ |
253 | if (offload_type != TX_OFFLOAD_CSUM) |
254 | break; |
255 | |
256 | *l4_offload = SCTP_OFFLOAD_ENABLE; |
257 | *l4_len = sizeof(struct sctphdr); |
258 | *offset = TRANSPORT_OFFSET(l4->hdr, skb); |
259 | break; |
260 | |
261 | default: |
262 | break; |
263 | } |
264 | } |
265 | |
266 | static __sum16 csum_magic(union hinic_l3 *ip, unsigned short proto) |
267 | { |
268 | return (ip->v4->version == 4) ? |
269 | csum_tcpudp_magic(saddr: ip->v4->saddr, daddr: ip->v4->daddr, len: 0, proto, sum: 0) : |
270 | csum_ipv6_magic(saddr: &ip->v6->saddr, daddr: &ip->v6->daddr, len: 0, proto, sum: 0); |
271 | } |
272 | |
273 | static int offload_tso(struct hinic_sq_task *task, u32 *queue_info, |
274 | struct sk_buff *skb) |
275 | { |
276 | u32 offset, l4_len, ip_identify, network_hdr_len; |
277 | enum hinic_l3_offload_type l3_offload; |
278 | enum hinic_l4_offload_type l4_offload; |
279 | union hinic_l3 ip; |
280 | union hinic_l4 l4; |
281 | u8 l4_proto; |
282 | |
283 | if (!skb_is_gso(skb)) |
284 | return 0; |
285 | |
286 | if (skb_cow_head(skb, headroom: 0) < 0) |
287 | return -EPROTONOSUPPORT; |
288 | |
289 | if (skb->encapsulation) { |
290 | u32 gso_type = skb_shinfo(skb)->gso_type; |
291 | u32 tunnel_type = 0; |
292 | u32 l4_tunnel_len; |
293 | |
294 | ip.hdr = skb_network_header(skb); |
295 | l4.hdr = skb_transport_header(skb); |
296 | network_hdr_len = skb_inner_network_header_len(skb); |
297 | |
298 | if (ip.v4->version == 4) { |
299 | ip.v4->tot_len = 0; |
300 | l3_offload = IPV4_PKT_WITH_CHKSUM_OFFLOAD; |
301 | } else if (ip.v4->version == 6) { |
302 | l3_offload = IPV6_PKT; |
303 | } else { |
304 | l3_offload = 0; |
305 | } |
306 | |
307 | hinic_task_set_outter_l3(task, l3_type: l3_offload, |
308 | network_len: skb_network_header_len(skb)); |
309 | |
310 | if (gso_type & SKB_GSO_UDP_TUNNEL_CSUM) { |
311 | l4.udp->check = ~csum_magic(ip: &ip, IPPROTO_UDP); |
312 | tunnel_type = TUNNEL_UDP_CSUM; |
313 | } else if (gso_type & SKB_GSO_UDP_TUNNEL) { |
314 | tunnel_type = TUNNEL_UDP_NO_CSUM; |
315 | } |
316 | |
317 | l4_tunnel_len = skb_inner_network_offset(skb) - |
318 | skb_transport_offset(skb); |
319 | hinic_task_set_tunnel_l4(task, l4_type: tunnel_type, tunnel_len: l4_tunnel_len); |
320 | |
321 | ip.hdr = skb_inner_network_header(skb); |
322 | l4.hdr = skb_inner_transport_header(skb); |
323 | } else { |
324 | ip.hdr = skb_network_header(skb); |
325 | l4.hdr = skb_transport_header(skb); |
326 | network_hdr_len = skb_network_header_len(skb); |
327 | } |
328 | |
329 | /* initialize inner IP header fields */ |
330 | if (ip.v4->version == 4) |
331 | ip.v4->tot_len = 0; |
332 | else |
333 | ip.v6->payload_len = 0; |
334 | |
335 | get_inner_l3_l4_type(skb, ip: &ip, l4: &l4, offload_type: TX_OFFLOAD_TSO, l3_type: &l3_offload, |
336 | l4_proto: &l4_proto); |
337 | |
338 | hinic_task_set_inner_l3(task, l3_type: l3_offload, network_len: network_hdr_len); |
339 | |
340 | ip_identify = 0; |
341 | if (l4_proto == IPPROTO_TCP) |
342 | l4.tcp->check = ~csum_magic(ip: &ip, IPPROTO_TCP); |
343 | |
344 | get_inner_l4_info(skb, l4: &l4, offload_type: TX_OFFLOAD_TSO, l4_proto, l4_offload: &l4_offload, |
345 | l4_len: &l4_len, offset: &offset); |
346 | |
347 | hinic_set_tso_inner_l4(task, queue_info, l4_offload, l4_len, offset, |
348 | ip_ident: ip_identify, skb_shinfo(skb)->gso_size); |
349 | |
350 | return 1; |
351 | } |
352 | |
353 | static int offload_csum(struct hinic_sq_task *task, u32 *queue_info, |
354 | struct sk_buff *skb) |
355 | { |
356 | enum hinic_l4_offload_type l4_offload; |
357 | u32 offset, l4_len, network_hdr_len; |
358 | enum hinic_l3_offload_type l3_type; |
359 | u32 tunnel_type = NOT_TUNNEL; |
360 | union hinic_l3 ip; |
361 | union hinic_l4 l4; |
362 | u8 l4_proto; |
363 | |
364 | if (skb->ip_summed != CHECKSUM_PARTIAL) |
365 | return 0; |
366 | |
367 | if (skb->encapsulation) { |
368 | u32 l4_tunnel_len; |
369 | |
370 | tunnel_type = TUNNEL_UDP_NO_CSUM; |
371 | ip.hdr = skb_network_header(skb); |
372 | |
373 | if (ip.v4->version == 4) { |
374 | l3_type = IPV4_PKT_NO_CHKSUM_OFFLOAD; |
375 | l4_proto = ip.v4->protocol; |
376 | } else if (ip.v4->version == 6) { |
377 | unsigned char *exthdr; |
378 | __be16 frag_off; |
379 | |
380 | l3_type = IPV6_PKT; |
381 | tunnel_type = TUNNEL_UDP_CSUM; |
382 | exthdr = ip.hdr + sizeof(*ip.v6); |
383 | l4_proto = ip.v6->nexthdr; |
384 | l4.hdr = skb_transport_header(skb); |
385 | if (l4.hdr != exthdr) |
386 | ipv6_skip_exthdr(skb, start: exthdr - skb->data, |
387 | nexthdrp: &l4_proto, frag_offp: &frag_off); |
388 | } else { |
389 | l3_type = L3TYPE_UNKNOWN; |
390 | l4_proto = IPPROTO_RAW; |
391 | } |
392 | |
393 | hinic_task_set_outter_l3(task, l3_type, |
394 | network_len: skb_network_header_len(skb)); |
395 | |
396 | switch (l4_proto) { |
397 | case IPPROTO_UDP: |
398 | l4_tunnel_len = skb_inner_network_offset(skb) - |
399 | skb_transport_offset(skb); |
400 | ip.hdr = skb_inner_network_header(skb); |
401 | l4.hdr = skb_inner_transport_header(skb); |
402 | network_hdr_len = skb_inner_network_header_len(skb); |
403 | break; |
404 | case IPPROTO_IPIP: |
405 | case IPPROTO_IPV6: |
406 | tunnel_type = NOT_TUNNEL; |
407 | l4_tunnel_len = 0; |
408 | |
409 | ip.hdr = skb_inner_network_header(skb); |
410 | l4.hdr = skb_transport_header(skb); |
411 | network_hdr_len = skb_network_header_len(skb); |
412 | break; |
413 | default: |
414 | /* Unsupported tunnel packet, disable csum offload */ |
415 | skb_checksum_help(skb); |
416 | return 0; |
417 | } |
418 | |
419 | hinic_task_set_tunnel_l4(task, l4_type: tunnel_type, tunnel_len: l4_tunnel_len); |
420 | } else { |
421 | ip.hdr = skb_network_header(skb); |
422 | l4.hdr = skb_transport_header(skb); |
423 | network_hdr_len = skb_network_header_len(skb); |
424 | } |
425 | |
426 | get_inner_l3_l4_type(skb, ip: &ip, l4: &l4, offload_type: TX_OFFLOAD_CSUM, l3_type: &l3_type, |
427 | l4_proto: &l4_proto); |
428 | |
429 | hinic_task_set_inner_l3(task, l3_type, network_len: network_hdr_len); |
430 | |
431 | get_inner_l4_info(skb, l4: &l4, offload_type: TX_OFFLOAD_CSUM, l4_proto, l4_offload: &l4_offload, |
432 | l4_len: &l4_len, offset: &offset); |
433 | |
434 | hinic_set_cs_inner_l4(task, queue_info, l4_offload, l4_len, offset); |
435 | |
436 | return 1; |
437 | } |
438 | |
439 | static void offload_vlan(struct hinic_sq_task *task, u32 *queue_info, |
440 | u16 vlan_tag, u16 vlan_pri) |
441 | { |
442 | task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(vlan_tag, VLAN_TAG) | |
443 | HINIC_SQ_TASK_INFO0_SET(1U, VLAN_OFFLOAD); |
444 | |
445 | *queue_info |= HINIC_SQ_CTRL_SET(vlan_pri, QUEUE_INFO_PRI); |
446 | } |
447 | |
448 | static int hinic_tx_offload(struct sk_buff *skb, struct hinic_sq_task *task, |
449 | u32 *queue_info) |
450 | { |
451 | enum hinic_offload_type offload = 0; |
452 | u16 vlan_tag; |
453 | int enabled; |
454 | |
455 | enabled = offload_tso(task, queue_info, skb); |
456 | if (enabled > 0) { |
457 | offload |= TX_OFFLOAD_TSO; |
458 | } else if (enabled == 0) { |
459 | enabled = offload_csum(task, queue_info, skb); |
460 | if (enabled) |
461 | offload |= TX_OFFLOAD_CSUM; |
462 | } else { |
463 | return -EPROTONOSUPPORT; |
464 | } |
465 | |
466 | if (unlikely(skb_vlan_tag_present(skb))) { |
467 | vlan_tag = skb_vlan_tag_get(skb); |
468 | offload_vlan(task, queue_info, vlan_tag, |
469 | vlan_pri: vlan_tag >> VLAN_PRIO_SHIFT); |
470 | offload |= TX_OFFLOAD_VLAN; |
471 | } |
472 | |
473 | if (offload) |
474 | hinic_task_set_l2hdr(task, len: skb_network_offset(skb)); |
475 | |
476 | /* payload offset should not more than 221 */ |
477 | if (HINIC_SQ_CTRL_GET(*queue_info, QUEUE_INFO_PLDOFF) > |
478 | MAX_PAYLOAD_OFFSET) { |
479 | return -EPROTONOSUPPORT; |
480 | } |
481 | |
482 | /* mss should not less than 80 */ |
483 | if (HINIC_SQ_CTRL_GET(*queue_info, QUEUE_INFO_MSS) < HINIC_MSS_MIN) { |
484 | *queue_info = HINIC_SQ_CTRL_CLEAR(*queue_info, QUEUE_INFO_MSS); |
485 | *queue_info |= HINIC_SQ_CTRL_SET(HINIC_MSS_MIN, QUEUE_INFO_MSS); |
486 | } |
487 | |
488 | return 0; |
489 | } |
490 | |
491 | netdev_tx_t hinic_lb_xmit_frame(struct sk_buff *skb, struct net_device *netdev) |
492 | { |
493 | struct hinic_dev *nic_dev = netdev_priv(dev: netdev); |
494 | u16 prod_idx, q_id = skb->queue_mapping; |
495 | struct netdev_queue *netdev_txq; |
496 | int nr_sges, err = NETDEV_TX_OK; |
497 | struct hinic_sq_wqe *sq_wqe; |
498 | unsigned int wqe_size; |
499 | struct hinic_txq *txq; |
500 | struct hinic_qp *qp; |
501 | |
502 | txq = &nic_dev->txqs[q_id]; |
503 | qp = container_of(txq->sq, struct hinic_qp, sq); |
504 | nr_sges = skb_shinfo(skb)->nr_frags + 1; |
505 | |
506 | err = tx_map_skb(nic_dev, skb, sges: txq->sges); |
507 | if (err) |
508 | goto skb_error; |
509 | |
510 | wqe_size = HINIC_SQ_WQE_SIZE(nr_sges); |
511 | |
512 | sq_wqe = hinic_sq_get_wqe(sq: txq->sq, wqe_size, prod_idx: &prod_idx); |
513 | if (!sq_wqe) { |
514 | netif_stop_subqueue(dev: netdev, queue_index: qp->q_id); |
515 | |
516 | sq_wqe = hinic_sq_get_wqe(sq: txq->sq, wqe_size, prod_idx: &prod_idx); |
517 | if (sq_wqe) { |
518 | netif_wake_subqueue(dev: nic_dev->netdev, queue_index: qp->q_id); |
519 | goto process_sq_wqe; |
520 | } |
521 | |
522 | tx_unmap_skb(nic_dev, skb, sges: txq->sges); |
523 | |
524 | u64_stats_update_begin(syncp: &txq->txq_stats.syncp); |
525 | txq->txq_stats.tx_busy++; |
526 | u64_stats_update_end(syncp: &txq->txq_stats.syncp); |
527 | err = NETDEV_TX_BUSY; |
528 | wqe_size = 0; |
529 | goto flush_skbs; |
530 | } |
531 | |
532 | process_sq_wqe: |
533 | hinic_sq_prepare_wqe(sq: txq->sq, wqe: sq_wqe, sges: txq->sges, nr_sges); |
534 | hinic_sq_write_wqe(sq: txq->sq, prod_idx, wqe: sq_wqe, skb, wqe_size); |
535 | |
536 | flush_skbs: |
537 | netdev_txq = netdev_get_tx_queue(dev: netdev, index: q_id); |
538 | if ((!netdev_xmit_more()) || (netif_xmit_stopped(dev_queue: netdev_txq))) |
539 | hinic_sq_write_db(sq: txq->sq, prod_idx, wqe_size, cos: 0); |
540 | |
541 | return err; |
542 | |
543 | skb_error: |
544 | dev_kfree_skb_any(skb); |
545 | u64_stats_update_begin(syncp: &txq->txq_stats.syncp); |
546 | txq->txq_stats.tx_dropped++; |
547 | u64_stats_update_end(syncp: &txq->txq_stats.syncp); |
548 | |
549 | return NETDEV_TX_OK; |
550 | } |
551 | |
552 | netdev_tx_t hinic_xmit_frame(struct sk_buff *skb, struct net_device *netdev) |
553 | { |
554 | struct hinic_dev *nic_dev = netdev_priv(dev: netdev); |
555 | u16 prod_idx, q_id = skb->queue_mapping; |
556 | struct netdev_queue *netdev_txq; |
557 | int nr_sges, err = NETDEV_TX_OK; |
558 | struct hinic_sq_wqe *sq_wqe; |
559 | unsigned int wqe_size; |
560 | struct hinic_txq *txq; |
561 | struct hinic_qp *qp; |
562 | |
563 | txq = &nic_dev->txqs[q_id]; |
564 | qp = container_of(txq->sq, struct hinic_qp, sq); |
565 | |
566 | if (skb->len < MIN_SKB_LEN) { |
567 | if (skb_pad(skb, MIN_SKB_LEN - skb->len)) { |
568 | netdev_err(dev: netdev, format: "Failed to pad skb\n" ); |
569 | goto update_error_stats; |
570 | } |
571 | |
572 | skb->len = MIN_SKB_LEN; |
573 | } |
574 | |
575 | nr_sges = skb_shinfo(skb)->nr_frags + 1; |
576 | if (nr_sges > 17) { |
577 | u64_stats_update_begin(syncp: &txq->txq_stats.syncp); |
578 | txq->txq_stats.big_frags_pkts++; |
579 | u64_stats_update_end(syncp: &txq->txq_stats.syncp); |
580 | } |
581 | |
582 | if (nr_sges > txq->max_sges) { |
583 | netdev_err(dev: netdev, format: "Too many Tx sges\n" ); |
584 | goto skb_error; |
585 | } |
586 | |
587 | err = tx_map_skb(nic_dev, skb, sges: txq->sges); |
588 | if (err) |
589 | goto skb_error; |
590 | |
591 | wqe_size = HINIC_SQ_WQE_SIZE(nr_sges); |
592 | |
593 | sq_wqe = hinic_sq_get_wqe(sq: txq->sq, wqe_size, prod_idx: &prod_idx); |
594 | if (!sq_wqe) { |
595 | netif_stop_subqueue(dev: netdev, queue_index: qp->q_id); |
596 | |
597 | /* Check for the case free_tx_poll is called in another cpu |
598 | * and we stopped the subqueue after free_tx_poll check. |
599 | */ |
600 | sq_wqe = hinic_sq_get_wqe(sq: txq->sq, wqe_size, prod_idx: &prod_idx); |
601 | if (sq_wqe) { |
602 | netif_wake_subqueue(dev: nic_dev->netdev, queue_index: qp->q_id); |
603 | goto process_sq_wqe; |
604 | } |
605 | |
606 | tx_unmap_skb(nic_dev, skb, sges: txq->sges); |
607 | |
608 | u64_stats_update_begin(syncp: &txq->txq_stats.syncp); |
609 | txq->txq_stats.tx_busy++; |
610 | u64_stats_update_end(syncp: &txq->txq_stats.syncp); |
611 | err = NETDEV_TX_BUSY; |
612 | wqe_size = 0; |
613 | goto flush_skbs; |
614 | } |
615 | |
616 | process_sq_wqe: |
617 | hinic_sq_prepare_wqe(sq: txq->sq, wqe: sq_wqe, sges: txq->sges, nr_sges); |
618 | |
619 | err = hinic_tx_offload(skb, task: &sq_wqe->task, queue_info: &sq_wqe->ctrl.queue_info); |
620 | if (err) |
621 | goto offload_error; |
622 | |
623 | hinic_sq_write_wqe(sq: txq->sq, prod_idx, wqe: sq_wqe, skb, wqe_size); |
624 | |
625 | flush_skbs: |
626 | netdev_txq = netdev_get_tx_queue(dev: netdev, index: q_id); |
627 | if ((!netdev_xmit_more()) || (netif_xmit_stopped(dev_queue: netdev_txq))) |
628 | hinic_sq_write_db(sq: txq->sq, prod_idx, wqe_size, cos: 0); |
629 | |
630 | return err; |
631 | |
632 | offload_error: |
633 | hinic_sq_return_wqe(sq: txq->sq, wqe_size); |
634 | tx_unmap_skb(nic_dev, skb, sges: txq->sges); |
635 | |
636 | skb_error: |
637 | dev_kfree_skb_any(skb); |
638 | |
639 | update_error_stats: |
640 | u64_stats_update_begin(syncp: &txq->txq_stats.syncp); |
641 | txq->txq_stats.tx_dropped++; |
642 | u64_stats_update_end(syncp: &txq->txq_stats.syncp); |
643 | |
644 | return NETDEV_TX_OK; |
645 | } |
646 | |
647 | /** |
648 | * tx_free_skb - unmap and free skb |
649 | * @nic_dev: nic device |
650 | * @skb: the skb |
651 | * @sges: the sges that are connected to the skb |
652 | **/ |
653 | static void tx_free_skb(struct hinic_dev *nic_dev, struct sk_buff *skb, |
654 | struct hinic_sge *sges) |
655 | { |
656 | tx_unmap_skb(nic_dev, skb, sges); |
657 | |
658 | dev_kfree_skb_any(skb); |
659 | } |
660 | |
661 | /** |
662 | * free_all_tx_skbs - free all skbs in tx queue |
663 | * @txq: tx queue |
664 | **/ |
665 | static void free_all_tx_skbs(struct hinic_txq *txq) |
666 | { |
667 | struct hinic_dev *nic_dev = netdev_priv(dev: txq->netdev); |
668 | struct hinic_sq *sq = txq->sq; |
669 | struct hinic_sq_wqe *sq_wqe; |
670 | unsigned int wqe_size; |
671 | struct sk_buff *skb; |
672 | int nr_sges; |
673 | u16 ci; |
674 | |
675 | while ((sq_wqe = hinic_sq_read_wqebb(sq, skb: &skb, wqe_size: &wqe_size, cons_idx: &ci))) { |
676 | sq_wqe = hinic_sq_read_wqe(sq, skb: &skb, wqe_size, cons_idx: &ci); |
677 | if (!sq_wqe) |
678 | break; |
679 | |
680 | nr_sges = skb_shinfo(skb)->nr_frags + 1; |
681 | |
682 | hinic_sq_get_sges(wqe: sq_wqe, sges: txq->free_sges, nr_sges); |
683 | |
684 | hinic_sq_put_wqe(sq, wqe_size); |
685 | |
686 | tx_free_skb(nic_dev, skb, sges: txq->free_sges); |
687 | } |
688 | } |
689 | |
690 | /** |
691 | * free_tx_poll - free finished tx skbs in tx queue that connected to napi |
692 | * @napi: napi |
693 | * @budget: number of tx |
694 | * |
695 | * Return 0 - Success, negative - Failure |
696 | **/ |
697 | static int free_tx_poll(struct napi_struct *napi, int budget) |
698 | { |
699 | struct hinic_txq *txq = container_of(napi, struct hinic_txq, napi); |
700 | struct hinic_qp *qp = container_of(txq->sq, struct hinic_qp, sq); |
701 | struct hinic_dev *nic_dev = netdev_priv(dev: txq->netdev); |
702 | struct netdev_queue *netdev_txq; |
703 | struct hinic_sq *sq = txq->sq; |
704 | struct hinic_wq *wq = sq->wq; |
705 | struct hinic_sq_wqe *sq_wqe; |
706 | unsigned int wqe_size; |
707 | int nr_sges, pkts = 0; |
708 | struct sk_buff *skb; |
709 | u64 tx_bytes = 0; |
710 | u16 hw_ci, sw_ci; |
711 | |
712 | do { |
713 | hw_ci = HW_CONS_IDX(sq) & wq->mask; |
714 | |
715 | dma_rmb(); |
716 | |
717 | /* Reading a WQEBB to get real WQE size and consumer index. */ |
718 | sq_wqe = hinic_sq_read_wqebb(sq, skb: &skb, wqe_size: &wqe_size, cons_idx: &sw_ci); |
719 | if (!sq_wqe || |
720 | (((hw_ci - sw_ci) & wq->mask) * wq->wqebb_size < wqe_size)) |
721 | break; |
722 | |
723 | /* If this WQE have multiple WQEBBs, we will read again to get |
724 | * full size WQE. |
725 | */ |
726 | if (wqe_size > wq->wqebb_size) { |
727 | sq_wqe = hinic_sq_read_wqe(sq, skb: &skb, wqe_size, cons_idx: &sw_ci); |
728 | if (unlikely(!sq_wqe)) |
729 | break; |
730 | } |
731 | |
732 | tx_bytes += skb->len; |
733 | pkts++; |
734 | |
735 | nr_sges = skb_shinfo(skb)->nr_frags + 1; |
736 | |
737 | hinic_sq_get_sges(wqe: sq_wqe, sges: txq->free_sges, nr_sges); |
738 | |
739 | hinic_sq_put_wqe(sq, wqe_size); |
740 | |
741 | tx_free_skb(nic_dev, skb, sges: txq->free_sges); |
742 | } while (pkts < budget); |
743 | |
744 | if (__netif_subqueue_stopped(dev: nic_dev->netdev, queue_index: qp->q_id) && |
745 | hinic_get_sq_free_wqebbs(sq) >= HINIC_MIN_TX_NUM_WQEBBS(sq)) { |
746 | netdev_txq = netdev_get_tx_queue(dev: txq->netdev, index: qp->q_id); |
747 | |
748 | __netif_tx_lock(txq: netdev_txq, smp_processor_id()); |
749 | if (!netif_testing(dev: nic_dev->netdev)) |
750 | netif_wake_subqueue(dev: nic_dev->netdev, queue_index: qp->q_id); |
751 | |
752 | __netif_tx_unlock(txq: netdev_txq); |
753 | |
754 | u64_stats_update_begin(syncp: &txq->txq_stats.syncp); |
755 | txq->txq_stats.tx_wake++; |
756 | u64_stats_update_end(syncp: &txq->txq_stats.syncp); |
757 | } |
758 | |
759 | u64_stats_update_begin(syncp: &txq->txq_stats.syncp); |
760 | txq->txq_stats.bytes += tx_bytes; |
761 | txq->txq_stats.pkts += pkts; |
762 | u64_stats_update_end(syncp: &txq->txq_stats.syncp); |
763 | |
764 | if (pkts < budget) { |
765 | napi_complete(n: napi); |
766 | if (!HINIC_IS_VF(nic_dev->hwdev->hwif)) |
767 | hinic_hwdev_set_msix_state(hwdev: nic_dev->hwdev, |
768 | msix_index: sq->msix_entry, |
769 | flag: HINIC_MSIX_ENABLE); |
770 | |
771 | return pkts; |
772 | } |
773 | |
774 | return budget; |
775 | } |
776 | |
777 | static irqreturn_t tx_irq(int irq, void *data) |
778 | { |
779 | struct hinic_txq *txq = data; |
780 | struct hinic_dev *nic_dev; |
781 | |
782 | nic_dev = netdev_priv(dev: txq->netdev); |
783 | |
784 | if (!HINIC_IS_VF(nic_dev->hwdev->hwif)) |
785 | /* Disable the interrupt until napi will be completed */ |
786 | hinic_hwdev_set_msix_state(hwdev: nic_dev->hwdev, |
787 | msix_index: txq->sq->msix_entry, |
788 | flag: HINIC_MSIX_DISABLE); |
789 | |
790 | hinic_hwdev_msix_cnt_set(hwdev: nic_dev->hwdev, msix_index: txq->sq->msix_entry); |
791 | |
792 | napi_schedule(n: &txq->napi); |
793 | return IRQ_HANDLED; |
794 | } |
795 | |
796 | static int tx_request_irq(struct hinic_txq *txq) |
797 | { |
798 | struct hinic_dev *nic_dev = netdev_priv(dev: txq->netdev); |
799 | struct hinic_msix_config interrupt_info = {0}; |
800 | struct hinic_intr_coal_info *intr_coal = NULL; |
801 | struct hinic_hwdev *hwdev = nic_dev->hwdev; |
802 | struct hinic_hwif *hwif = hwdev->hwif; |
803 | struct pci_dev *pdev = hwif->pdev; |
804 | struct hinic_sq *sq = txq->sq; |
805 | struct hinic_qp *qp; |
806 | int err; |
807 | |
808 | qp = container_of(sq, struct hinic_qp, sq); |
809 | |
810 | netif_napi_add_weight(dev: txq->netdev, napi: &txq->napi, poll: free_tx_poll, |
811 | weight: nic_dev->tx_weight); |
812 | |
813 | hinic_hwdev_msix_set(hwdev: nic_dev->hwdev, msix_index: sq->msix_entry, |
814 | TX_IRQ_NO_PENDING, TX_IRQ_NO_COALESC, |
815 | TX_IRQ_NO_LLI_TIMER, TX_IRQ_NO_CREDIT, |
816 | TX_IRQ_NO_RESEND_TIMER); |
817 | |
818 | intr_coal = &nic_dev->tx_intr_coalesce[qp->q_id]; |
819 | interrupt_info.msix_index = sq->msix_entry; |
820 | interrupt_info.coalesce_timer_cnt = intr_coal->coalesce_timer_cfg; |
821 | interrupt_info.pending_cnt = intr_coal->pending_limt; |
822 | interrupt_info.resend_timer_cnt = intr_coal->resend_timer_cfg; |
823 | |
824 | err = hinic_set_interrupt_cfg(hwdev, interrupt_info: &interrupt_info); |
825 | if (err) { |
826 | netif_err(nic_dev, drv, txq->netdev, |
827 | "Failed to set TX interrupt coalescing attribute\n" ); |
828 | netif_napi_del(napi: &txq->napi); |
829 | return err; |
830 | } |
831 | |
832 | err = request_irq(irq: sq->irq, handler: tx_irq, flags: 0, name: txq->irq_name, dev: txq); |
833 | if (err) { |
834 | dev_err(&pdev->dev, "Failed to request Tx irq\n" ); |
835 | netif_napi_del(napi: &txq->napi); |
836 | return err; |
837 | } |
838 | |
839 | return 0; |
840 | } |
841 | |
842 | static void tx_free_irq(struct hinic_txq *txq) |
843 | { |
844 | struct hinic_sq *sq = txq->sq; |
845 | |
846 | free_irq(sq->irq, txq); |
847 | netif_napi_del(napi: &txq->napi); |
848 | } |
849 | |
850 | /** |
851 | * hinic_init_txq - Initialize the Tx Queue |
852 | * @txq: Logical Tx Queue |
853 | * @sq: Hardware Tx Queue to connect the Logical queue with |
854 | * @netdev: network device to connect the Logical queue with |
855 | * |
856 | * Return 0 - Success, negative - Failure |
857 | **/ |
858 | int hinic_init_txq(struct hinic_txq *txq, struct hinic_sq *sq, |
859 | struct net_device *netdev) |
860 | { |
861 | struct hinic_qp *qp = container_of(sq, struct hinic_qp, sq); |
862 | struct hinic_dev *nic_dev = netdev_priv(dev: netdev); |
863 | struct hinic_hwdev *hwdev = nic_dev->hwdev; |
864 | int err; |
865 | |
866 | txq->netdev = netdev; |
867 | txq->sq = sq; |
868 | |
869 | txq_stats_init(txq); |
870 | |
871 | txq->max_sges = HINIC_MAX_SQ_BUFDESCS; |
872 | |
873 | txq->sges = devm_kcalloc(dev: &netdev->dev, n: txq->max_sges, |
874 | size: sizeof(*txq->sges), GFP_KERNEL); |
875 | if (!txq->sges) |
876 | return -ENOMEM; |
877 | |
878 | txq->free_sges = devm_kcalloc(dev: &netdev->dev, n: txq->max_sges, |
879 | size: sizeof(*txq->free_sges), GFP_KERNEL); |
880 | if (!txq->free_sges) { |
881 | err = -ENOMEM; |
882 | goto err_alloc_free_sges; |
883 | } |
884 | |
885 | txq->irq_name = devm_kasprintf(dev: &netdev->dev, GFP_KERNEL, fmt: "%s_txq%d" , |
886 | netdev->name, qp->q_id); |
887 | if (!txq->irq_name) { |
888 | err = -ENOMEM; |
889 | goto err_alloc_irqname; |
890 | } |
891 | |
892 | err = hinic_hwdev_hw_ci_addr_set(hwdev, sq, CI_UPDATE_NO_PENDING, |
893 | CI_UPDATE_NO_COALESC); |
894 | if (err) |
895 | goto err_hw_ci; |
896 | |
897 | err = tx_request_irq(txq); |
898 | if (err) { |
899 | netdev_err(dev: netdev, format: "Failed to request Tx irq\n" ); |
900 | goto err_req_tx_irq; |
901 | } |
902 | |
903 | return 0; |
904 | |
905 | err_req_tx_irq: |
906 | err_hw_ci: |
907 | devm_kfree(dev: &netdev->dev, p: txq->irq_name); |
908 | |
909 | err_alloc_irqname: |
910 | devm_kfree(dev: &netdev->dev, p: txq->free_sges); |
911 | |
912 | err_alloc_free_sges: |
913 | devm_kfree(dev: &netdev->dev, p: txq->sges); |
914 | return err; |
915 | } |
916 | |
917 | /** |
918 | * hinic_clean_txq - Clean the Tx Queue |
919 | * @txq: Logical Tx Queue |
920 | **/ |
921 | void hinic_clean_txq(struct hinic_txq *txq) |
922 | { |
923 | struct net_device *netdev = txq->netdev; |
924 | |
925 | tx_free_irq(txq); |
926 | |
927 | free_all_tx_skbs(txq); |
928 | |
929 | devm_kfree(dev: &netdev->dev, p: txq->irq_name); |
930 | devm_kfree(dev: &netdev->dev, p: txq->free_sges); |
931 | devm_kfree(dev: &netdev->dev, p: txq->sges); |
932 | } |
933 | |