1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Marvell RVU Ethernet driver |
3 | * |
4 | * Copyright (C) 2020 Marvell. |
5 | * |
6 | */ |
7 | |
8 | #include <linux/etherdevice.h> |
9 | #include <net/ip.h> |
10 | #include <net/tso.h> |
11 | #include <linux/bpf.h> |
12 | #include <linux/bpf_trace.h> |
13 | #include <net/ip6_checksum.h> |
14 | |
15 | #include "otx2_reg.h" |
16 | #include "otx2_common.h" |
17 | #include "otx2_struct.h" |
18 | #include "otx2_txrx.h" |
19 | #include "otx2_ptp.h" |
20 | #include "cn10k.h" |
21 | |
22 | #define CQE_ADDR(CQ, idx) ((CQ)->cqe_base + ((CQ)->cqe_size * (idx))) |
23 | #define PTP_PORT 0x13F |
24 | /* PTPv2 header Original Timestamp starts at byte offset 34 and |
25 | * contains 6 byte seconds field and 4 byte nano seconds field. |
26 | */ |
27 | #define PTP_SYNC_SEC_OFFSET 34 |
28 | |
29 | static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf, |
30 | struct bpf_prog *prog, |
31 | struct nix_cqe_rx_s *cqe, |
32 | struct otx2_cq_queue *cq, |
33 | bool *need_xdp_flush); |
34 | |
35 | static int otx2_nix_cq_op_status(struct otx2_nic *pfvf, |
36 | struct otx2_cq_queue *cq) |
37 | { |
38 | u64 incr = (u64)(cq->cq_idx) << 32; |
39 | u64 status; |
40 | |
41 | status = otx2_atomic64_fetch_add(incr, pfvf->cq_op_addr); |
42 | |
43 | if (unlikely(status & BIT_ULL(CQ_OP_STAT_OP_ERR) || |
44 | status & BIT_ULL(CQ_OP_STAT_CQ_ERR))) { |
45 | dev_err(pfvf->dev, "CQ stopped due to error" ); |
46 | return -EINVAL; |
47 | } |
48 | |
49 | cq->cq_tail = status & 0xFFFFF; |
50 | cq->cq_head = (status >> 20) & 0xFFFFF; |
51 | if (cq->cq_tail < cq->cq_head) |
52 | cq->pend_cqe = (cq->cqe_cnt - cq->cq_head) + |
53 | cq->cq_tail; |
54 | else |
55 | cq->pend_cqe = cq->cq_tail - cq->cq_head; |
56 | |
57 | return 0; |
58 | } |
59 | |
60 | static struct nix_cqe_hdr_s *otx2_get_next_cqe(struct otx2_cq_queue *cq) |
61 | { |
62 | struct nix_cqe_hdr_s *cqe_hdr; |
63 | |
64 | cqe_hdr = (struct nix_cqe_hdr_s *)CQE_ADDR(cq, cq->cq_head); |
65 | if (cqe_hdr->cqe_type == NIX_XQE_TYPE_INVALID) |
66 | return NULL; |
67 | |
68 | cq->cq_head++; |
69 | cq->cq_head &= (cq->cqe_cnt - 1); |
70 | |
71 | return cqe_hdr; |
72 | } |
73 | |
74 | static unsigned int frag_num(unsigned int i) |
75 | { |
76 | #ifdef __BIG_ENDIAN |
77 | return (i & ~3) + 3 - (i & 3); |
78 | #else |
79 | return i; |
80 | #endif |
81 | } |
82 | |
83 | static dma_addr_t otx2_dma_map_skb_frag(struct otx2_nic *pfvf, |
84 | struct sk_buff *skb, int seg, int *len) |
85 | { |
86 | const skb_frag_t *frag; |
87 | struct page *page; |
88 | int offset; |
89 | |
90 | /* First segment is always skb->data */ |
91 | if (!seg) { |
92 | page = virt_to_page(skb->data); |
93 | offset = offset_in_page(skb->data); |
94 | *len = skb_headlen(skb); |
95 | } else { |
96 | frag = &skb_shinfo(skb)->frags[seg - 1]; |
97 | page = skb_frag_page(frag); |
98 | offset = skb_frag_off(frag); |
99 | *len = skb_frag_size(frag); |
100 | } |
101 | return otx2_dma_map_page(pfvf, page, offset, size: *len, dir: DMA_TO_DEVICE); |
102 | } |
103 | |
104 | static void otx2_dma_unmap_skb_frags(struct otx2_nic *pfvf, struct sg_list *sg) |
105 | { |
106 | int seg; |
107 | |
108 | for (seg = 0; seg < sg->num_segs; seg++) { |
109 | otx2_dma_unmap_page(pfvf, addr: sg->dma_addr[seg], |
110 | size: sg->size[seg], dir: DMA_TO_DEVICE); |
111 | } |
112 | sg->num_segs = 0; |
113 | } |
114 | |
115 | static void otx2_xdp_snd_pkt_handler(struct otx2_nic *pfvf, |
116 | struct otx2_snd_queue *sq, |
117 | struct nix_cqe_tx_s *cqe) |
118 | { |
119 | struct nix_send_comp_s *snd_comp = &cqe->comp; |
120 | struct sg_list *sg; |
121 | struct page *page; |
122 | u64 pa; |
123 | |
124 | sg = &sq->sg[snd_comp->sqe_id]; |
125 | |
126 | pa = otx2_iova_to_phys(iommu_domain: pfvf->iommu_domain, dma_addr: sg->dma_addr[0]); |
127 | otx2_dma_unmap_page(pfvf, addr: sg->dma_addr[0], |
128 | size: sg->size[0], dir: DMA_TO_DEVICE); |
129 | page = virt_to_page(phys_to_virt(pa)); |
130 | put_page(page); |
131 | } |
132 | |
133 | static void otx2_snd_pkt_handler(struct otx2_nic *pfvf, |
134 | struct otx2_cq_queue *cq, |
135 | struct otx2_snd_queue *sq, |
136 | struct nix_cqe_tx_s *cqe, |
137 | int budget, int *tx_pkts, int *tx_bytes) |
138 | { |
139 | struct nix_send_comp_s *snd_comp = &cqe->comp; |
140 | struct skb_shared_hwtstamps ts; |
141 | struct sk_buff *skb = NULL; |
142 | u64 timestamp, tsns; |
143 | struct sg_list *sg; |
144 | int err; |
145 | |
146 | if (unlikely(snd_comp->status) && netif_msg_tx_err(pfvf)) |
147 | net_err_ratelimited("%s: TX%d: Error in send CQ status:%x\n" , |
148 | pfvf->netdev->name, cq->cint_idx, |
149 | snd_comp->status); |
150 | |
151 | sg = &sq->sg[snd_comp->sqe_id]; |
152 | skb = (struct sk_buff *)sg->skb; |
153 | if (unlikely(!skb)) |
154 | return; |
155 | |
156 | if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) { |
157 | timestamp = ((u64 *)sq->timestamps->base)[snd_comp->sqe_id]; |
158 | if (timestamp != 1) { |
159 | timestamp = pfvf->ptp->convert_tx_ptp_tstmp(timestamp); |
160 | err = otx2_ptp_tstamp2time(pfvf, tstamp: timestamp, tsns: &tsns); |
161 | if (!err) { |
162 | memset(&ts, 0, sizeof(ts)); |
163 | ts.hwtstamp = ns_to_ktime(ns: tsns); |
164 | skb_tstamp_tx(orig_skb: skb, hwtstamps: &ts); |
165 | } |
166 | } |
167 | } |
168 | |
169 | *tx_bytes += skb->len; |
170 | (*tx_pkts)++; |
171 | otx2_dma_unmap_skb_frags(pfvf, sg); |
172 | napi_consume_skb(skb, budget); |
173 | sg->skb = (u64)NULL; |
174 | } |
175 | |
176 | static void otx2_set_rxtstamp(struct otx2_nic *pfvf, |
177 | struct sk_buff *skb, void *data) |
178 | { |
179 | u64 timestamp, tsns; |
180 | int err; |
181 | |
182 | if (!(pfvf->flags & OTX2_FLAG_RX_TSTAMP_ENABLED)) |
183 | return; |
184 | |
185 | timestamp = pfvf->ptp->convert_rx_ptp_tstmp(*(u64 *)data); |
186 | /* The first 8 bytes is the timestamp */ |
187 | err = otx2_ptp_tstamp2time(pfvf, tstamp: timestamp, tsns: &tsns); |
188 | if (err) |
189 | return; |
190 | |
191 | skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ns: tsns); |
192 | } |
193 | |
194 | static bool otx2_skb_add_frag(struct otx2_nic *pfvf, struct sk_buff *skb, |
195 | u64 iova, int len, struct nix_rx_parse_s *parse, |
196 | int qidx) |
197 | { |
198 | struct page *page; |
199 | int off = 0; |
200 | void *va; |
201 | |
202 | va = phys_to_virt(address: otx2_iova_to_phys(iommu_domain: pfvf->iommu_domain, dma_addr: iova)); |
203 | |
204 | if (likely(!skb_shinfo(skb)->nr_frags)) { |
205 | /* Check if data starts at some nonzero offset |
206 | * from the start of the buffer. For now the |
207 | * only possible offset is 8 bytes in the case |
208 | * where packet is prepended by a timestamp. |
209 | */ |
210 | if (parse->laptr) { |
211 | otx2_set_rxtstamp(pfvf, skb, data: va); |
212 | off = OTX2_HW_TIMESTAMP_LEN; |
213 | } |
214 | } |
215 | |
216 | page = virt_to_page(va); |
217 | if (likely(skb_shinfo(skb)->nr_frags < MAX_SKB_FRAGS)) { |
218 | skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, |
219 | off: va - page_address(page) + off, |
220 | size: len - off, truesize: pfvf->rbsize); |
221 | return true; |
222 | } |
223 | |
224 | /* If more than MAX_SKB_FRAGS fragments are received then |
225 | * give back those buffer pointers to hardware for reuse. |
226 | */ |
227 | pfvf->hw_ops->aura_freeptr(pfvf, qidx, iova & ~0x07ULL); |
228 | |
229 | return false; |
230 | } |
231 | |
232 | static void otx2_set_rxhash(struct otx2_nic *pfvf, |
233 | struct nix_cqe_rx_s *cqe, struct sk_buff *skb) |
234 | { |
235 | enum pkt_hash_types hash_type = PKT_HASH_TYPE_NONE; |
236 | struct otx2_rss_info *; |
237 | u32 hash = 0; |
238 | |
239 | if (!(pfvf->netdev->features & NETIF_F_RXHASH)) |
240 | return; |
241 | |
242 | rss = &pfvf->hw.rss_info; |
243 | if (rss->flowkey_cfg) { |
244 | if (rss->flowkey_cfg & |
245 | ~(NIX_FLOW_KEY_TYPE_IPV4 | NIX_FLOW_KEY_TYPE_IPV6)) |
246 | hash_type = PKT_HASH_TYPE_L4; |
247 | else |
248 | hash_type = PKT_HASH_TYPE_L3; |
249 | hash = cqe->hdr.flow_tag; |
250 | } |
251 | skb_set_hash(skb, hash, type: hash_type); |
252 | } |
253 | |
254 | static void otx2_free_rcv_seg(struct otx2_nic *pfvf, struct nix_cqe_rx_s *cqe, |
255 | int qidx) |
256 | { |
257 | struct nix_rx_sg_s *sg = &cqe->sg; |
258 | void *end, *start; |
259 | u64 *seg_addr; |
260 | int seg; |
261 | |
262 | start = (void *)sg; |
263 | end = start + ((cqe->parse.desc_sizem1 + 1) * 16); |
264 | while (start < end) { |
265 | sg = (struct nix_rx_sg_s *)start; |
266 | seg_addr = &sg->seg_addr; |
267 | for (seg = 0; seg < sg->segs; seg++, seg_addr++) |
268 | pfvf->hw_ops->aura_freeptr(pfvf, qidx, |
269 | *seg_addr & ~0x07ULL); |
270 | start += sizeof(*sg); |
271 | } |
272 | } |
273 | |
274 | static bool otx2_check_rcv_errors(struct otx2_nic *pfvf, |
275 | struct nix_cqe_rx_s *cqe, int qidx) |
276 | { |
277 | struct otx2_drv_stats *stats = &pfvf->hw.drv_stats; |
278 | struct nix_rx_parse_s *parse = &cqe->parse; |
279 | |
280 | if (netif_msg_rx_err(pfvf)) |
281 | netdev_err(dev: pfvf->netdev, |
282 | format: "RQ%d: Error pkt with errlev:0x%x errcode:0x%x\n" , |
283 | qidx, parse->errlev, parse->errcode); |
284 | |
285 | if (parse->errlev == NPC_ERRLVL_RE) { |
286 | switch (parse->errcode) { |
287 | case ERRCODE_FCS: |
288 | case ERRCODE_FCS_RCV: |
289 | atomic_inc(v: &stats->rx_fcs_errs); |
290 | break; |
291 | case ERRCODE_UNDERSIZE: |
292 | atomic_inc(v: &stats->rx_undersize_errs); |
293 | break; |
294 | case ERRCODE_OVERSIZE: |
295 | atomic_inc(v: &stats->rx_oversize_errs); |
296 | break; |
297 | case ERRCODE_OL2_LEN_MISMATCH: |
298 | atomic_inc(v: &stats->rx_len_errs); |
299 | break; |
300 | default: |
301 | atomic_inc(v: &stats->rx_other_errs); |
302 | break; |
303 | } |
304 | } else if (parse->errlev == NPC_ERRLVL_NIX) { |
305 | switch (parse->errcode) { |
306 | case ERRCODE_OL3_LEN: |
307 | case ERRCODE_OL4_LEN: |
308 | case ERRCODE_IL3_LEN: |
309 | case ERRCODE_IL4_LEN: |
310 | atomic_inc(v: &stats->rx_len_errs); |
311 | break; |
312 | case ERRCODE_OL4_CSUM: |
313 | case ERRCODE_IL4_CSUM: |
314 | atomic_inc(v: &stats->rx_csum_errs); |
315 | break; |
316 | default: |
317 | atomic_inc(v: &stats->rx_other_errs); |
318 | break; |
319 | } |
320 | } else { |
321 | atomic_inc(v: &stats->rx_other_errs); |
322 | /* For now ignore all the NPC parser errors and |
323 | * pass the packets to stack. |
324 | */ |
325 | return false; |
326 | } |
327 | |
328 | /* If RXALL is enabled pass on packets to stack. */ |
329 | if (pfvf->netdev->features & NETIF_F_RXALL) |
330 | return false; |
331 | |
332 | /* Free buffer back to pool */ |
333 | if (cqe->sg.segs) |
334 | otx2_free_rcv_seg(pfvf, cqe, qidx); |
335 | return true; |
336 | } |
337 | |
338 | static void otx2_rcv_pkt_handler(struct otx2_nic *pfvf, |
339 | struct napi_struct *napi, |
340 | struct otx2_cq_queue *cq, |
341 | struct nix_cqe_rx_s *cqe, bool *need_xdp_flush) |
342 | { |
343 | struct nix_rx_parse_s *parse = &cqe->parse; |
344 | struct nix_rx_sg_s *sg = &cqe->sg; |
345 | struct sk_buff *skb = NULL; |
346 | void *end, *start; |
347 | u64 *seg_addr; |
348 | u16 *seg_size; |
349 | int seg; |
350 | |
351 | if (unlikely(parse->errlev || parse->errcode)) { |
352 | if (otx2_check_rcv_errors(pfvf, cqe, qidx: cq->cq_idx)) |
353 | return; |
354 | } |
355 | |
356 | if (pfvf->xdp_prog) |
357 | if (otx2_xdp_rcv_pkt_handler(pfvf, prog: pfvf->xdp_prog, cqe, cq, need_xdp_flush)) |
358 | return; |
359 | |
360 | skb = napi_get_frags(napi); |
361 | if (unlikely(!skb)) |
362 | return; |
363 | |
364 | start = (void *)sg; |
365 | end = start + ((cqe->parse.desc_sizem1 + 1) * 16); |
366 | while (start < end) { |
367 | sg = (struct nix_rx_sg_s *)start; |
368 | seg_addr = &sg->seg_addr; |
369 | seg_size = (void *)sg; |
370 | for (seg = 0; seg < sg->segs; seg++, seg_addr++) { |
371 | if (otx2_skb_add_frag(pfvf, skb, iova: *seg_addr, |
372 | len: seg_size[seg], parse, qidx: cq->cq_idx)) |
373 | cq->pool_ptrs++; |
374 | } |
375 | start += sizeof(*sg); |
376 | } |
377 | otx2_set_rxhash(pfvf, cqe, skb); |
378 | |
379 | skb_record_rx_queue(skb, rx_queue: cq->cq_idx); |
380 | if (pfvf->netdev->features & NETIF_F_RXCSUM) |
381 | skb->ip_summed = CHECKSUM_UNNECESSARY; |
382 | |
383 | skb_mark_for_recycle(skb); |
384 | |
385 | napi_gro_frags(napi); |
386 | } |
387 | |
388 | static int otx2_rx_napi_handler(struct otx2_nic *pfvf, |
389 | struct napi_struct *napi, |
390 | struct otx2_cq_queue *cq, int budget) |
391 | { |
392 | bool need_xdp_flush = false; |
393 | struct nix_cqe_rx_s *cqe; |
394 | int processed_cqe = 0; |
395 | |
396 | if (cq->pend_cqe >= budget) |
397 | goto process_cqe; |
398 | |
399 | if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe) |
400 | return 0; |
401 | |
402 | process_cqe: |
403 | while (likely(processed_cqe < budget) && cq->pend_cqe) { |
404 | cqe = (struct nix_cqe_rx_s *)CQE_ADDR(cq, cq->cq_head); |
405 | if (cqe->hdr.cqe_type == NIX_XQE_TYPE_INVALID || |
406 | !cqe->sg.seg_addr) { |
407 | if (!processed_cqe) |
408 | return 0; |
409 | break; |
410 | } |
411 | cq->cq_head++; |
412 | cq->cq_head &= (cq->cqe_cnt - 1); |
413 | |
414 | otx2_rcv_pkt_handler(pfvf, napi, cq, cqe, need_xdp_flush: &need_xdp_flush); |
415 | |
416 | cqe->hdr.cqe_type = NIX_XQE_TYPE_INVALID; |
417 | cqe->sg.seg_addr = 0x00; |
418 | processed_cqe++; |
419 | cq->pend_cqe--; |
420 | } |
421 | if (need_xdp_flush) |
422 | xdp_do_flush(); |
423 | |
424 | /* Free CQEs to HW */ |
425 | otx2_write64(nic: pfvf, NIX_LF_CQ_OP_DOOR, |
426 | val: ((u64)cq->cq_idx << 32) | processed_cqe); |
427 | |
428 | return processed_cqe; |
429 | } |
430 | |
431 | int otx2_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq) |
432 | { |
433 | struct otx2_nic *pfvf = dev; |
434 | int cnt = cq->pool_ptrs; |
435 | dma_addr_t bufptr; |
436 | |
437 | while (cq->pool_ptrs) { |
438 | if (otx2_alloc_buffer(pfvf, cq, dma: &bufptr)) |
439 | break; |
440 | otx2_aura_freeptr(dev: pfvf, aura: cq->cq_idx, buf: bufptr + OTX2_HEAD_ROOM); |
441 | cq->pool_ptrs--; |
442 | } |
443 | |
444 | return cnt - cq->pool_ptrs; |
445 | } |
446 | |
447 | static int otx2_tx_napi_handler(struct otx2_nic *pfvf, |
448 | struct otx2_cq_queue *cq, int budget) |
449 | { |
450 | int tx_pkts = 0, tx_bytes = 0, qidx; |
451 | struct otx2_snd_queue *sq; |
452 | struct nix_cqe_tx_s *cqe; |
453 | int processed_cqe = 0; |
454 | |
455 | if (cq->pend_cqe >= budget) |
456 | goto process_cqe; |
457 | |
458 | if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe) |
459 | return 0; |
460 | |
461 | process_cqe: |
462 | qidx = cq->cq_idx - pfvf->hw.rx_queues; |
463 | sq = &pfvf->qset.sq[qidx]; |
464 | |
465 | while (likely(processed_cqe < budget) && cq->pend_cqe) { |
466 | cqe = (struct nix_cqe_tx_s *)otx2_get_next_cqe(cq); |
467 | if (unlikely(!cqe)) { |
468 | if (!processed_cqe) |
469 | return 0; |
470 | break; |
471 | } |
472 | |
473 | qidx = cq->cq_idx - pfvf->hw.rx_queues; |
474 | |
475 | if (cq->cq_type == CQ_XDP) |
476 | otx2_xdp_snd_pkt_handler(pfvf, sq, cqe); |
477 | else |
478 | otx2_snd_pkt_handler(pfvf, cq, sq: &pfvf->qset.sq[qidx], |
479 | cqe, budget, tx_pkts: &tx_pkts, tx_bytes: &tx_bytes); |
480 | |
481 | cqe->hdr.cqe_type = NIX_XQE_TYPE_INVALID; |
482 | processed_cqe++; |
483 | cq->pend_cqe--; |
484 | |
485 | sq->cons_head++; |
486 | sq->cons_head &= (sq->sqe_cnt - 1); |
487 | } |
488 | |
489 | /* Free CQEs to HW */ |
490 | otx2_write64(nic: pfvf, NIX_LF_CQ_OP_DOOR, |
491 | val: ((u64)cq->cq_idx << 32) | processed_cqe); |
492 | |
493 | if (likely(tx_pkts)) { |
494 | struct netdev_queue *txq; |
495 | |
496 | qidx = cq->cq_idx - pfvf->hw.rx_queues; |
497 | |
498 | if (qidx >= pfvf->hw.tx_queues) |
499 | qidx -= pfvf->hw.xdp_queues; |
500 | txq = netdev_get_tx_queue(dev: pfvf->netdev, index: qidx); |
501 | netdev_tx_completed_queue(dev_queue: txq, pkts: tx_pkts, bytes: tx_bytes); |
502 | /* Check if queue was stopped earlier due to ring full */ |
503 | smp_mb(); |
504 | if (netif_tx_queue_stopped(dev_queue: txq) && |
505 | netif_carrier_ok(dev: pfvf->netdev)) |
506 | netif_tx_wake_queue(dev_queue: txq); |
507 | } |
508 | return 0; |
509 | } |
510 | |
511 | static void otx2_adjust_adaptive_coalese(struct otx2_nic *pfvf, struct otx2_cq_poll *cq_poll) |
512 | { |
513 | struct dim_sample dim_sample; |
514 | u64 rx_frames, rx_bytes; |
515 | u64 tx_frames, tx_bytes; |
516 | |
517 | rx_frames = OTX2_GET_RX_STATS(RX_BCAST) + OTX2_GET_RX_STATS(RX_MCAST) + |
518 | OTX2_GET_RX_STATS(RX_UCAST); |
519 | rx_bytes = OTX2_GET_RX_STATS(RX_OCTS); |
520 | tx_bytes = OTX2_GET_TX_STATS(TX_OCTS); |
521 | tx_frames = OTX2_GET_TX_STATS(TX_UCAST); |
522 | |
523 | dim_update_sample(event_ctr: pfvf->napi_events, |
524 | packets: rx_frames + tx_frames, |
525 | bytes: rx_bytes + tx_bytes, |
526 | s: &dim_sample); |
527 | net_dim(dim: &cq_poll->dim, end_sample: dim_sample); |
528 | } |
529 | |
530 | int otx2_napi_handler(struct napi_struct *napi, int budget) |
531 | { |
532 | struct otx2_cq_queue *rx_cq = NULL; |
533 | struct otx2_cq_poll *cq_poll; |
534 | int workdone = 0, cq_idx, i; |
535 | struct otx2_cq_queue *cq; |
536 | struct otx2_qset *qset; |
537 | struct otx2_nic *pfvf; |
538 | int filled_cnt = -1; |
539 | |
540 | cq_poll = container_of(napi, struct otx2_cq_poll, napi); |
541 | pfvf = (struct otx2_nic *)cq_poll->dev; |
542 | qset = &pfvf->qset; |
543 | |
544 | for (i = 0; i < CQS_PER_CINT; i++) { |
545 | cq_idx = cq_poll->cq_ids[i]; |
546 | if (unlikely(cq_idx == CINT_INVALID_CQ)) |
547 | continue; |
548 | cq = &qset->cq[cq_idx]; |
549 | if (cq->cq_type == CQ_RX) { |
550 | rx_cq = cq; |
551 | workdone += otx2_rx_napi_handler(pfvf, napi, |
552 | cq, budget); |
553 | } else { |
554 | workdone += otx2_tx_napi_handler(pfvf, cq, budget); |
555 | } |
556 | } |
557 | |
558 | if (rx_cq && rx_cq->pool_ptrs) |
559 | filled_cnt = pfvf->hw_ops->refill_pool_ptrs(pfvf, rx_cq); |
560 | /* Clear the IRQ */ |
561 | otx2_write64(nic: pfvf, NIX_LF_CINTX_INT(cq_poll->cint_idx), BIT_ULL(0)); |
562 | |
563 | if (workdone < budget && napi_complete_done(n: napi, work_done: workdone)) { |
564 | /* If interface is going down, don't re-enable IRQ */ |
565 | if (pfvf->flags & OTX2_FLAG_INTF_DOWN) |
566 | return workdone; |
567 | |
568 | /* Adjust irq coalese using net_dim */ |
569 | if (pfvf->flags & OTX2_FLAG_ADPTV_INT_COAL_ENABLED) |
570 | otx2_adjust_adaptive_coalese(pfvf, cq_poll); |
571 | |
572 | if (unlikely(!filled_cnt)) { |
573 | struct refill_work *work; |
574 | struct delayed_work *dwork; |
575 | |
576 | work = &pfvf->refill_wrk[cq->cq_idx]; |
577 | dwork = &work->pool_refill_work; |
578 | /* Schedule a task if no other task is running */ |
579 | if (!cq->refill_task_sched) { |
580 | work->napi = napi; |
581 | cq->refill_task_sched = true; |
582 | schedule_delayed_work(dwork, |
583 | delay: msecs_to_jiffies(m: 100)); |
584 | } |
585 | } else { |
586 | /* Re-enable interrupts */ |
587 | otx2_write64(nic: pfvf, |
588 | NIX_LF_CINTX_ENA_W1S(cq_poll->cint_idx), |
589 | BIT_ULL(0)); |
590 | } |
591 | } |
592 | return workdone; |
593 | } |
594 | |
595 | void otx2_sqe_flush(void *dev, struct otx2_snd_queue *sq, |
596 | int size, int qidx) |
597 | { |
598 | u64 status; |
599 | |
600 | /* Packet data stores should finish before SQE is flushed to HW */ |
601 | dma_wmb(); |
602 | |
603 | do { |
604 | memcpy(sq->lmt_addr, sq->sqe_base, size); |
605 | status = otx2_lmt_flush(sq->io_addr); |
606 | } while (status == 0); |
607 | |
608 | sq->head++; |
609 | sq->head &= (sq->sqe_cnt - 1); |
610 | } |
611 | |
612 | #define MAX_SEGS_PER_SG 3 |
613 | /* Add SQE scatter/gather subdescriptor structure */ |
614 | static bool otx2_sqe_add_sg(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, |
615 | struct sk_buff *skb, int num_segs, int *offset) |
616 | { |
617 | struct nix_sqe_sg_s *sg = NULL; |
618 | u64 dma_addr, *iova = NULL; |
619 | u16 *sg_lens = NULL; |
620 | int seg, len; |
621 | |
622 | sq->sg[sq->head].num_segs = 0; |
623 | |
624 | for (seg = 0; seg < num_segs; seg++) { |
625 | if ((seg % MAX_SEGS_PER_SG) == 0) { |
626 | sg = (struct nix_sqe_sg_s *)(sq->sqe_base + *offset); |
627 | sg->ld_type = NIX_SEND_LDTYPE_LDD; |
628 | sg->subdc = NIX_SUBDC_SG; |
629 | sg->segs = 0; |
630 | sg_lens = (void *)sg; |
631 | iova = (void *)sg + sizeof(*sg); |
632 | /* Next subdc always starts at a 16byte boundary. |
633 | * So if sg->segs is whether 2 or 3, offset += 16bytes. |
634 | */ |
635 | if ((num_segs - seg) >= (MAX_SEGS_PER_SG - 1)) |
636 | *offset += sizeof(*sg) + (3 * sizeof(u64)); |
637 | else |
638 | *offset += sizeof(*sg) + sizeof(u64); |
639 | } |
640 | dma_addr = otx2_dma_map_skb_frag(pfvf, skb, seg, len: &len); |
641 | if (dma_mapping_error(dev: pfvf->dev, dma_addr)) |
642 | return false; |
643 | |
644 | sg_lens[frag_num(i: seg % MAX_SEGS_PER_SG)] = len; |
645 | sg->segs++; |
646 | *iova++ = dma_addr; |
647 | |
648 | /* Save DMA mapping info for later unmapping */ |
649 | sq->sg[sq->head].dma_addr[seg] = dma_addr; |
650 | sq->sg[sq->head].size[seg] = len; |
651 | sq->sg[sq->head].num_segs++; |
652 | } |
653 | |
654 | sq->sg[sq->head].skb = (u64)skb; |
655 | return true; |
656 | } |
657 | |
658 | /* Add SQE extended header subdescriptor */ |
659 | static void otx2_sqe_add_ext(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, |
660 | struct sk_buff *skb, int *offset) |
661 | { |
662 | struct nix_sqe_ext_s *ext; |
663 | |
664 | ext = (struct nix_sqe_ext_s *)(sq->sqe_base + *offset); |
665 | ext->subdc = NIX_SUBDC_EXT; |
666 | if (skb_shinfo(skb)->gso_size) { |
667 | ext->lso = 1; |
668 | ext->lso_sb = skb_tcp_all_headers(skb); |
669 | ext->lso_mps = skb_shinfo(skb)->gso_size; |
670 | |
671 | /* Only TSOv4 and TSOv6 GSO offloads are supported */ |
672 | if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) { |
673 | ext->lso_format = pfvf->hw.lso_tsov4_idx; |
674 | |
675 | /* HW adds payload size to 'ip_hdr->tot_len' while |
676 | * sending TSO segment, hence set payload length |
677 | * in IP header of the packet to just header length. |
678 | */ |
679 | ip_hdr(skb)->tot_len = |
680 | htons(ext->lso_sb - skb_network_offset(skb)); |
681 | } else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) { |
682 | ext->lso_format = pfvf->hw.lso_tsov6_idx; |
683 | ipv6_hdr(skb)->payload_len = htons(tcp_hdrlen(skb)); |
684 | } else if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { |
685 | __be16 l3_proto = vlan_get_protocol(skb); |
686 | struct udphdr *udph = udp_hdr(skb); |
687 | u16 iplen; |
688 | |
689 | ext->lso_sb = skb_transport_offset(skb) + |
690 | sizeof(struct udphdr); |
691 | |
692 | /* HW adds payload size to length fields in IP and |
693 | * UDP headers while segmentation, hence adjust the |
694 | * lengths to just header sizes. |
695 | */ |
696 | iplen = htons(ext->lso_sb - skb_network_offset(skb)); |
697 | if (l3_proto == htons(ETH_P_IP)) { |
698 | ip_hdr(skb)->tot_len = iplen; |
699 | ext->lso_format = pfvf->hw.lso_udpv4_idx; |
700 | } else { |
701 | ipv6_hdr(skb)->payload_len = iplen; |
702 | ext->lso_format = pfvf->hw.lso_udpv6_idx; |
703 | } |
704 | |
705 | udph->len = htons(sizeof(struct udphdr)); |
706 | } |
707 | } else if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) { |
708 | ext->tstmp = 1; |
709 | } |
710 | |
711 | #define OTX2_VLAN_PTR_OFFSET (ETH_HLEN - ETH_TLEN) |
712 | if (skb_vlan_tag_present(skb)) { |
713 | if (skb->vlan_proto == htons(ETH_P_8021Q)) { |
714 | ext->vlan1_ins_ena = 1; |
715 | ext->vlan1_ins_ptr = OTX2_VLAN_PTR_OFFSET; |
716 | ext->vlan1_ins_tci = skb_vlan_tag_get(skb); |
717 | } else if (skb->vlan_proto == htons(ETH_P_8021AD)) { |
718 | ext->vlan0_ins_ena = 1; |
719 | ext->vlan0_ins_ptr = OTX2_VLAN_PTR_OFFSET; |
720 | ext->vlan0_ins_tci = skb_vlan_tag_get(skb); |
721 | } |
722 | } |
723 | |
724 | *offset += sizeof(*ext); |
725 | } |
726 | |
727 | static void otx2_sqe_add_mem(struct otx2_snd_queue *sq, int *offset, |
728 | int alg, u64 iova, int ptp_offset, |
729 | u64 base_ns, bool udp_csum_crt) |
730 | { |
731 | struct nix_sqe_mem_s *mem; |
732 | |
733 | mem = (struct nix_sqe_mem_s *)(sq->sqe_base + *offset); |
734 | mem->subdc = NIX_SUBDC_MEM; |
735 | mem->alg = alg; |
736 | mem->wmem = 1; /* wait for the memory operation */ |
737 | mem->addr = iova; |
738 | |
739 | if (ptp_offset) { |
740 | mem->start_offset = ptp_offset; |
741 | mem->udp_csum_crt = !!udp_csum_crt; |
742 | mem->base_ns = base_ns; |
743 | mem->step_type = 1; |
744 | } |
745 | |
746 | *offset += sizeof(*mem); |
747 | } |
748 | |
749 | /* Add SQE header subdescriptor structure */ |
750 | static void otx2_sqe_add_hdr(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, |
751 | struct nix_sqe_hdr_s *sqe_hdr, |
752 | struct sk_buff *skb, u16 qidx) |
753 | { |
754 | int proto = 0; |
755 | |
756 | /* Check if SQE was framed before, if yes then no need to |
757 | * set these constants again and again. |
758 | */ |
759 | if (!sqe_hdr->total) { |
760 | /* Don't free Tx buffers to Aura */ |
761 | sqe_hdr->df = 1; |
762 | sqe_hdr->aura = sq->aura_id; |
763 | /* Post a CQE Tx after pkt transmission */ |
764 | sqe_hdr->pnc = 1; |
765 | sqe_hdr->sq = (qidx >= pfvf->hw.tx_queues) ? |
766 | qidx + pfvf->hw.xdp_queues : qidx; |
767 | } |
768 | sqe_hdr->total = skb->len; |
769 | /* Set SQE identifier which will be used later for freeing SKB */ |
770 | sqe_hdr->sqe_id = sq->head; |
771 | |
772 | /* Offload TCP/UDP checksum to HW */ |
773 | if (skb->ip_summed == CHECKSUM_PARTIAL) { |
774 | sqe_hdr->ol3ptr = skb_network_offset(skb); |
775 | sqe_hdr->ol4ptr = skb_transport_offset(skb); |
776 | /* get vlan protocol Ethertype */ |
777 | if (eth_type_vlan(ethertype: skb->protocol)) |
778 | skb->protocol = vlan_get_protocol(skb); |
779 | |
780 | if (skb->protocol == htons(ETH_P_IP)) { |
781 | proto = ip_hdr(skb)->protocol; |
782 | /* In case of TSO, HW needs this to be explicitly set. |
783 | * So set this always, instead of adding a check. |
784 | */ |
785 | sqe_hdr->ol3type = NIX_SENDL3TYPE_IP4_CKSUM; |
786 | } else if (skb->protocol == htons(ETH_P_IPV6)) { |
787 | proto = ipv6_hdr(skb)->nexthdr; |
788 | sqe_hdr->ol3type = NIX_SENDL3TYPE_IP6; |
789 | } |
790 | |
791 | if (proto == IPPROTO_TCP) |
792 | sqe_hdr->ol4type = NIX_SENDL4TYPE_TCP_CKSUM; |
793 | else if (proto == IPPROTO_UDP) |
794 | sqe_hdr->ol4type = NIX_SENDL4TYPE_UDP_CKSUM; |
795 | } |
796 | } |
797 | |
798 | static int otx2_dma_map_tso_skb(struct otx2_nic *pfvf, |
799 | struct otx2_snd_queue *sq, |
800 | struct sk_buff *skb, int sqe, int hdr_len) |
801 | { |
802 | int num_segs = skb_shinfo(skb)->nr_frags + 1; |
803 | struct sg_list *sg = &sq->sg[sqe]; |
804 | u64 dma_addr; |
805 | int seg, len; |
806 | |
807 | sg->num_segs = 0; |
808 | |
809 | /* Get payload length at skb->data */ |
810 | len = skb_headlen(skb) - hdr_len; |
811 | |
812 | for (seg = 0; seg < num_segs; seg++) { |
813 | /* Skip skb->data, if there is no payload */ |
814 | if (!seg && !len) |
815 | continue; |
816 | dma_addr = otx2_dma_map_skb_frag(pfvf, skb, seg, len: &len); |
817 | if (dma_mapping_error(dev: pfvf->dev, dma_addr)) |
818 | goto unmap; |
819 | |
820 | /* Save DMA mapping info for later unmapping */ |
821 | sg->dma_addr[sg->num_segs] = dma_addr; |
822 | sg->size[sg->num_segs] = len; |
823 | sg->num_segs++; |
824 | } |
825 | return 0; |
826 | unmap: |
827 | otx2_dma_unmap_skb_frags(pfvf, sg); |
828 | return -EINVAL; |
829 | } |
830 | |
831 | static u64 otx2_tso_frag_dma_addr(struct otx2_snd_queue *sq, |
832 | struct sk_buff *skb, int seg, |
833 | u64 seg_addr, int hdr_len, int sqe) |
834 | { |
835 | struct sg_list *sg = &sq->sg[sqe]; |
836 | const skb_frag_t *frag; |
837 | int offset; |
838 | |
839 | if (seg < 0) |
840 | return sg->dma_addr[0] + (seg_addr - (u64)skb->data); |
841 | |
842 | frag = &skb_shinfo(skb)->frags[seg]; |
843 | offset = seg_addr - (u64)skb_frag_address(frag); |
844 | if (skb_headlen(skb) - hdr_len) |
845 | seg++; |
846 | return sg->dma_addr[seg] + offset; |
847 | } |
848 | |
849 | static void otx2_sqe_tso_add_sg(struct otx2_snd_queue *sq, |
850 | struct sg_list *list, int *offset) |
851 | { |
852 | struct nix_sqe_sg_s *sg = NULL; |
853 | u16 *sg_lens = NULL; |
854 | u64 *iova = NULL; |
855 | int seg; |
856 | |
857 | /* Add SG descriptors with buffer addresses */ |
858 | for (seg = 0; seg < list->num_segs; seg++) { |
859 | if ((seg % MAX_SEGS_PER_SG) == 0) { |
860 | sg = (struct nix_sqe_sg_s *)(sq->sqe_base + *offset); |
861 | sg->ld_type = NIX_SEND_LDTYPE_LDD; |
862 | sg->subdc = NIX_SUBDC_SG; |
863 | sg->segs = 0; |
864 | sg_lens = (void *)sg; |
865 | iova = (void *)sg + sizeof(*sg); |
866 | /* Next subdc always starts at a 16byte boundary. |
867 | * So if sg->segs is whether 2 or 3, offset += 16bytes. |
868 | */ |
869 | if ((list->num_segs - seg) >= (MAX_SEGS_PER_SG - 1)) |
870 | *offset += sizeof(*sg) + (3 * sizeof(u64)); |
871 | else |
872 | *offset += sizeof(*sg) + sizeof(u64); |
873 | } |
874 | sg_lens[frag_num(i: seg % MAX_SEGS_PER_SG)] = list->size[seg]; |
875 | *iova++ = list->dma_addr[seg]; |
876 | sg->segs++; |
877 | } |
878 | } |
879 | |
880 | static void otx2_sq_append_tso(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, |
881 | struct sk_buff *skb, u16 qidx) |
882 | { |
883 | struct netdev_queue *txq = netdev_get_tx_queue(dev: pfvf->netdev, index: qidx); |
884 | int hdr_len, tcp_data, seg_len, pkt_len, offset; |
885 | struct nix_sqe_hdr_s *sqe_hdr; |
886 | int first_sqe = sq->head; |
887 | struct sg_list list; |
888 | struct tso_t tso; |
889 | |
890 | hdr_len = tso_start(skb, tso: &tso); |
891 | |
892 | /* Map SKB's fragments to DMA. |
893 | * It's done here to avoid mapping for every TSO segment's packet. |
894 | */ |
895 | if (otx2_dma_map_tso_skb(pfvf, sq, skb, sqe: first_sqe, hdr_len)) { |
896 | dev_kfree_skb_any(skb); |
897 | return; |
898 | } |
899 | |
900 | netdev_tx_sent_queue(dev_queue: txq, bytes: skb->len); |
901 | |
902 | tcp_data = skb->len - hdr_len; |
903 | while (tcp_data > 0) { |
904 | char *hdr; |
905 | |
906 | seg_len = min_t(int, skb_shinfo(skb)->gso_size, tcp_data); |
907 | tcp_data -= seg_len; |
908 | |
909 | /* Set SQE's SEND_HDR */ |
910 | memset(sq->sqe_base, 0, sq->sqe_size); |
911 | sqe_hdr = (struct nix_sqe_hdr_s *)(sq->sqe_base); |
912 | otx2_sqe_add_hdr(pfvf, sq, sqe_hdr, skb, qidx); |
913 | offset = sizeof(*sqe_hdr); |
914 | |
915 | /* Add TSO segment's pkt header */ |
916 | hdr = sq->tso_hdrs->base + (sq->head * TSO_HEADER_SIZE); |
917 | tso_build_hdr(skb, hdr, tso: &tso, size: seg_len, is_last: tcp_data == 0); |
918 | list.dma_addr[0] = |
919 | sq->tso_hdrs->iova + (sq->head * TSO_HEADER_SIZE); |
920 | list.size[0] = hdr_len; |
921 | list.num_segs = 1; |
922 | |
923 | /* Add TSO segment's payload data fragments */ |
924 | pkt_len = hdr_len; |
925 | while (seg_len > 0) { |
926 | int size; |
927 | |
928 | size = min_t(int, tso.size, seg_len); |
929 | |
930 | list.size[list.num_segs] = size; |
931 | list.dma_addr[list.num_segs] = |
932 | otx2_tso_frag_dma_addr(sq, skb, |
933 | seg: tso.next_frag_idx - 1, |
934 | seg_addr: (u64)tso.data, hdr_len, |
935 | sqe: first_sqe); |
936 | list.num_segs++; |
937 | pkt_len += size; |
938 | seg_len -= size; |
939 | tso_build_data(skb, tso: &tso, size); |
940 | } |
941 | sqe_hdr->total = pkt_len; |
942 | otx2_sqe_tso_add_sg(sq, list: &list, offset: &offset); |
943 | |
944 | /* DMA mappings and skb needs to be freed only after last |
945 | * TSO segment is transmitted out. So set 'PNC' only for |
946 | * last segment. Also point last segment's sqe_id to first |
947 | * segment's SQE index where skb address and DMA mappings |
948 | * are saved. |
949 | */ |
950 | if (!tcp_data) { |
951 | sqe_hdr->pnc = 1; |
952 | sqe_hdr->sqe_id = first_sqe; |
953 | sq->sg[first_sqe].skb = (u64)skb; |
954 | } else { |
955 | sqe_hdr->pnc = 0; |
956 | } |
957 | |
958 | sqe_hdr->sizem1 = (offset / 16) - 1; |
959 | |
960 | /* Flush SQE to HW */ |
961 | pfvf->hw_ops->sqe_flush(pfvf, sq, offset, qidx); |
962 | } |
963 | } |
964 | |
965 | static bool is_hw_tso_supported(struct otx2_nic *pfvf, |
966 | struct sk_buff *skb) |
967 | { |
968 | int payload_len, last_seg_size; |
969 | |
970 | if (test_bit(HW_TSO, &pfvf->hw.cap_flag)) |
971 | return true; |
972 | |
973 | /* On 96xx A0, HW TSO not supported */ |
974 | if (!is_96xx_B0(pdev: pfvf->pdev)) |
975 | return false; |
976 | |
977 | /* HW has an issue due to which when the payload of the last LSO |
978 | * segment is shorter than 16 bytes, some header fields may not |
979 | * be correctly modified, hence don't offload such TSO segments. |
980 | */ |
981 | |
982 | payload_len = skb->len - skb_tcp_all_headers(skb); |
983 | last_seg_size = payload_len % skb_shinfo(skb)->gso_size; |
984 | if (last_seg_size && last_seg_size < 16) |
985 | return false; |
986 | |
987 | return true; |
988 | } |
989 | |
990 | static int otx2_get_sqe_count(struct otx2_nic *pfvf, struct sk_buff *skb) |
991 | { |
992 | if (!skb_shinfo(skb)->gso_size) |
993 | return 1; |
994 | |
995 | /* HW TSO */ |
996 | if (is_hw_tso_supported(pfvf, skb)) |
997 | return 1; |
998 | |
999 | /* SW TSO */ |
1000 | return skb_shinfo(skb)->gso_segs; |
1001 | } |
1002 | |
1003 | static bool otx2_validate_network_transport(struct sk_buff *skb) |
1004 | { |
1005 | if ((ip_hdr(skb)->protocol == IPPROTO_UDP) || |
1006 | (ipv6_hdr(skb)->nexthdr == IPPROTO_UDP)) { |
1007 | struct udphdr *udph = udp_hdr(skb); |
1008 | |
1009 | if (udph->source == htons(PTP_PORT) && |
1010 | udph->dest == htons(PTP_PORT)) |
1011 | return true; |
1012 | } |
1013 | |
1014 | return false; |
1015 | } |
1016 | |
1017 | static bool otx2_ptp_is_sync(struct sk_buff *skb, int *offset, bool *udp_csum_crt) |
1018 | { |
1019 | struct ethhdr *eth = (struct ethhdr *)(skb->data); |
1020 | u16 nix_offload_hlen = 0, inner_vhlen = 0; |
1021 | bool udp_hdr_present = false, is_sync; |
1022 | u8 *data = skb->data, *msgtype; |
1023 | __be16 proto = eth->h_proto; |
1024 | int network_depth = 0; |
1025 | |
1026 | /* NIX is programmed to offload outer VLAN header |
1027 | * in case of single vlan protocol field holds Network header ETH_IP/V6 |
1028 | * in case of stacked vlan protocol field holds Inner vlan (8100) |
1029 | */ |
1030 | if (skb->dev->features & NETIF_F_HW_VLAN_CTAG_TX && |
1031 | skb->dev->features & NETIF_F_HW_VLAN_STAG_TX) { |
1032 | if (skb->vlan_proto == htons(ETH_P_8021AD)) { |
1033 | /* Get vlan protocol */ |
1034 | proto = __vlan_get_protocol(skb, type: eth->h_proto, NULL); |
1035 | /* SKB APIs like skb_transport_offset does not include |
1036 | * offloaded vlan header length. Need to explicitly add |
1037 | * the length |
1038 | */ |
1039 | nix_offload_hlen = VLAN_HLEN; |
1040 | inner_vhlen = VLAN_HLEN; |
1041 | } else if (skb->vlan_proto == htons(ETH_P_8021Q)) { |
1042 | nix_offload_hlen = VLAN_HLEN; |
1043 | } |
1044 | } else if (eth_type_vlan(ethertype: eth->h_proto)) { |
1045 | proto = __vlan_get_protocol(skb, type: eth->h_proto, depth: &network_depth); |
1046 | } |
1047 | |
1048 | switch (ntohs(proto)) { |
1049 | case ETH_P_1588: |
1050 | if (network_depth) |
1051 | *offset = network_depth; |
1052 | else |
1053 | *offset = ETH_HLEN + nix_offload_hlen + |
1054 | inner_vhlen; |
1055 | break; |
1056 | case ETH_P_IP: |
1057 | case ETH_P_IPV6: |
1058 | if (!otx2_validate_network_transport(skb)) |
1059 | return false; |
1060 | |
1061 | *offset = nix_offload_hlen + skb_transport_offset(skb) + |
1062 | sizeof(struct udphdr); |
1063 | udp_hdr_present = true; |
1064 | |
1065 | } |
1066 | |
1067 | msgtype = data + *offset; |
1068 | /* Check PTP messageId is SYNC or not */ |
1069 | is_sync = !(*msgtype & 0xf); |
1070 | if (is_sync) |
1071 | *udp_csum_crt = udp_hdr_present; |
1072 | else |
1073 | *offset = 0; |
1074 | |
1075 | return is_sync; |
1076 | } |
1077 | |
1078 | static void otx2_set_txtstamp(struct otx2_nic *pfvf, struct sk_buff *skb, |
1079 | struct otx2_snd_queue *sq, int *offset) |
1080 | { |
1081 | struct ethhdr *eth = (struct ethhdr *)(skb->data); |
1082 | struct ptpv2_tstamp *origin_tstamp; |
1083 | bool udp_csum_crt = false; |
1084 | unsigned int udphoff; |
1085 | struct timespec64 ts; |
1086 | int ptp_offset = 0; |
1087 | __wsum skb_csum; |
1088 | u64 iova; |
1089 | |
1090 | if (unlikely(!skb_shinfo(skb)->gso_size && |
1091 | (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) { |
1092 | if (unlikely(pfvf->flags & OTX2_FLAG_PTP_ONESTEP_SYNC && |
1093 | otx2_ptp_is_sync(skb, &ptp_offset, &udp_csum_crt))) { |
1094 | origin_tstamp = (struct ptpv2_tstamp *) |
1095 | ((u8 *)skb->data + ptp_offset + |
1096 | PTP_SYNC_SEC_OFFSET); |
1097 | ts = ns_to_timespec64(nsec: pfvf->ptp->tstamp); |
1098 | origin_tstamp->seconds_msb = htons((ts.tv_sec >> 32) & 0xffff); |
1099 | origin_tstamp->seconds_lsb = htonl(ts.tv_sec & 0xffffffff); |
1100 | origin_tstamp->nanoseconds = htonl(ts.tv_nsec); |
1101 | /* Point to correction field in PTP packet */ |
1102 | ptp_offset += 8; |
1103 | |
1104 | /* When user disables hw checksum, stack calculates the csum, |
1105 | * but it does not cover ptp timestamp which is added later. |
1106 | * Recalculate the checksum manually considering the timestamp. |
1107 | */ |
1108 | if (udp_csum_crt) { |
1109 | struct udphdr *uh = udp_hdr(skb); |
1110 | |
1111 | if (skb->ip_summed != CHECKSUM_PARTIAL && uh->check != 0) { |
1112 | udphoff = skb_transport_offset(skb); |
1113 | uh->check = 0; |
1114 | skb_csum = skb_checksum(skb, offset: udphoff, len: skb->len - udphoff, |
1115 | csum: 0); |
1116 | if (ntohs(eth->h_proto) == ETH_P_IPV6) |
1117 | uh->check = csum_ipv6_magic(saddr: &ipv6_hdr(skb)->saddr, |
1118 | daddr: &ipv6_hdr(skb)->daddr, |
1119 | len: skb->len - udphoff, |
1120 | proto: ipv6_hdr(skb)->nexthdr, |
1121 | sum: skb_csum); |
1122 | else |
1123 | uh->check = csum_tcpudp_magic(saddr: ip_hdr(skb)->saddr, |
1124 | daddr: ip_hdr(skb)->daddr, |
1125 | len: skb->len - udphoff, |
1126 | IPPROTO_UDP, |
1127 | sum: skb_csum); |
1128 | } |
1129 | } |
1130 | } else { |
1131 | skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; |
1132 | } |
1133 | iova = sq->timestamps->iova + (sq->head * sizeof(u64)); |
1134 | otx2_sqe_add_mem(sq, offset, alg: NIX_SENDMEMALG_E_SETTSTMP, iova, |
1135 | ptp_offset, base_ns: pfvf->ptp->base_ns, udp_csum_crt); |
1136 | } else { |
1137 | skb_tx_timestamp(skb); |
1138 | } |
1139 | } |
1140 | |
1141 | bool otx2_sq_append_skb(struct net_device *netdev, struct otx2_snd_queue *sq, |
1142 | struct sk_buff *skb, u16 qidx) |
1143 | { |
1144 | struct netdev_queue *txq = netdev_get_tx_queue(dev: netdev, index: qidx); |
1145 | struct otx2_nic *pfvf = netdev_priv(dev: netdev); |
1146 | int offset, num_segs, free_desc; |
1147 | struct nix_sqe_hdr_s *sqe_hdr; |
1148 | |
1149 | /* Check if there is enough room between producer |
1150 | * and consumer index. |
1151 | */ |
1152 | free_desc = (sq->cons_head - sq->head - 1 + sq->sqe_cnt) & (sq->sqe_cnt - 1); |
1153 | if (free_desc < sq->sqe_thresh) |
1154 | return false; |
1155 | |
1156 | if (free_desc < otx2_get_sqe_count(pfvf, skb)) |
1157 | return false; |
1158 | |
1159 | num_segs = skb_shinfo(skb)->nr_frags + 1; |
1160 | |
1161 | /* If SKB doesn't fit in a single SQE, linearize it. |
1162 | * TODO: Consider adding JUMP descriptor instead. |
1163 | */ |
1164 | if (unlikely(num_segs > OTX2_MAX_FRAGS_IN_SQE)) { |
1165 | if (__skb_linearize(skb)) { |
1166 | dev_kfree_skb_any(skb); |
1167 | return true; |
1168 | } |
1169 | num_segs = skb_shinfo(skb)->nr_frags + 1; |
1170 | } |
1171 | |
1172 | if (skb_shinfo(skb)->gso_size && !is_hw_tso_supported(pfvf, skb)) { |
1173 | /* Insert vlan tag before giving pkt to tso */ |
1174 | if (skb_vlan_tag_present(skb)) |
1175 | skb = __vlan_hwaccel_push_inside(skb); |
1176 | otx2_sq_append_tso(pfvf, sq, skb, qidx); |
1177 | return true; |
1178 | } |
1179 | |
1180 | /* Set SQE's SEND_HDR. |
1181 | * Do not clear the first 64bit as it contains constant info. |
1182 | */ |
1183 | memset(sq->sqe_base + 8, 0, sq->sqe_size - 8); |
1184 | sqe_hdr = (struct nix_sqe_hdr_s *)(sq->sqe_base); |
1185 | otx2_sqe_add_hdr(pfvf, sq, sqe_hdr, skb, qidx); |
1186 | offset = sizeof(*sqe_hdr); |
1187 | |
1188 | /* Add extended header if needed */ |
1189 | otx2_sqe_add_ext(pfvf, sq, skb, offset: &offset); |
1190 | |
1191 | /* Add SG subdesc with data frags */ |
1192 | if (!otx2_sqe_add_sg(pfvf, sq, skb, num_segs, offset: &offset)) { |
1193 | otx2_dma_unmap_skb_frags(pfvf, sg: &sq->sg[sq->head]); |
1194 | return false; |
1195 | } |
1196 | |
1197 | otx2_set_txtstamp(pfvf, skb, sq, offset: &offset); |
1198 | |
1199 | sqe_hdr->sizem1 = (offset / 16) - 1; |
1200 | |
1201 | netdev_tx_sent_queue(dev_queue: txq, bytes: skb->len); |
1202 | |
1203 | /* Flush SQE to HW */ |
1204 | pfvf->hw_ops->sqe_flush(pfvf, sq, offset, qidx); |
1205 | |
1206 | return true; |
1207 | } |
1208 | EXPORT_SYMBOL(otx2_sq_append_skb); |
1209 | |
1210 | void otx2_cleanup_rx_cqes(struct otx2_nic *pfvf, struct otx2_cq_queue *cq, int qidx) |
1211 | { |
1212 | struct nix_cqe_rx_s *cqe; |
1213 | struct otx2_pool *pool; |
1214 | int processed_cqe = 0; |
1215 | u16 pool_id; |
1216 | u64 iova; |
1217 | |
1218 | if (pfvf->xdp_prog) |
1219 | xdp_rxq_info_unreg(xdp_rxq: &cq->xdp_rxq); |
1220 | |
1221 | if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe) |
1222 | return; |
1223 | |
1224 | pool_id = otx2_get_pool_idx(pfvf, type: AURA_NIX_RQ, idx: qidx); |
1225 | pool = &pfvf->qset.pool[pool_id]; |
1226 | |
1227 | while (cq->pend_cqe) { |
1228 | cqe = (struct nix_cqe_rx_s *)otx2_get_next_cqe(cq); |
1229 | processed_cqe++; |
1230 | cq->pend_cqe--; |
1231 | |
1232 | if (!cqe) |
1233 | continue; |
1234 | if (cqe->sg.segs > 1) { |
1235 | otx2_free_rcv_seg(pfvf, cqe, qidx: cq->cq_idx); |
1236 | continue; |
1237 | } |
1238 | iova = cqe->sg.seg_addr - OTX2_HEAD_ROOM; |
1239 | |
1240 | otx2_free_bufs(pfvf, pool, iova, size: pfvf->rbsize); |
1241 | } |
1242 | |
1243 | /* Free CQEs to HW */ |
1244 | otx2_write64(nic: pfvf, NIX_LF_CQ_OP_DOOR, |
1245 | val: ((u64)cq->cq_idx << 32) | processed_cqe); |
1246 | } |
1247 | |
1248 | void otx2_cleanup_tx_cqes(struct otx2_nic *pfvf, struct otx2_cq_queue *cq) |
1249 | { |
1250 | int tx_pkts = 0, tx_bytes = 0; |
1251 | struct sk_buff *skb = NULL; |
1252 | struct otx2_snd_queue *sq; |
1253 | struct nix_cqe_tx_s *cqe; |
1254 | struct netdev_queue *txq; |
1255 | int processed_cqe = 0; |
1256 | struct sg_list *sg; |
1257 | int qidx; |
1258 | |
1259 | qidx = cq->cq_idx - pfvf->hw.rx_queues; |
1260 | sq = &pfvf->qset.sq[qidx]; |
1261 | |
1262 | if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe) |
1263 | return; |
1264 | |
1265 | while (cq->pend_cqe) { |
1266 | cqe = (struct nix_cqe_tx_s *)otx2_get_next_cqe(cq); |
1267 | processed_cqe++; |
1268 | cq->pend_cqe--; |
1269 | |
1270 | if (!cqe) |
1271 | continue; |
1272 | sg = &sq->sg[cqe->comp.sqe_id]; |
1273 | skb = (struct sk_buff *)sg->skb; |
1274 | if (skb) { |
1275 | tx_bytes += skb->len; |
1276 | tx_pkts++; |
1277 | otx2_dma_unmap_skb_frags(pfvf, sg); |
1278 | dev_kfree_skb_any(skb); |
1279 | sg->skb = (u64)NULL; |
1280 | } |
1281 | } |
1282 | |
1283 | if (likely(tx_pkts)) { |
1284 | if (qidx >= pfvf->hw.tx_queues) |
1285 | qidx -= pfvf->hw.xdp_queues; |
1286 | txq = netdev_get_tx_queue(dev: pfvf->netdev, index: qidx); |
1287 | netdev_tx_completed_queue(dev_queue: txq, pkts: tx_pkts, bytes: tx_bytes); |
1288 | } |
1289 | /* Free CQEs to HW */ |
1290 | otx2_write64(nic: pfvf, NIX_LF_CQ_OP_DOOR, |
1291 | val: ((u64)cq->cq_idx << 32) | processed_cqe); |
1292 | } |
1293 | |
1294 | int otx2_rxtx_enable(struct otx2_nic *pfvf, bool enable) |
1295 | { |
1296 | struct msg_req *msg; |
1297 | int err; |
1298 | |
1299 | mutex_lock(&pfvf->mbox.lock); |
1300 | if (enable) |
1301 | msg = otx2_mbox_alloc_msg_nix_lf_start_rx(mbox: &pfvf->mbox); |
1302 | else |
1303 | msg = otx2_mbox_alloc_msg_nix_lf_stop_rx(mbox: &pfvf->mbox); |
1304 | |
1305 | if (!msg) { |
1306 | mutex_unlock(lock: &pfvf->mbox.lock); |
1307 | return -ENOMEM; |
1308 | } |
1309 | |
1310 | err = otx2_sync_mbox_msg(mbox: &pfvf->mbox); |
1311 | mutex_unlock(lock: &pfvf->mbox.lock); |
1312 | return err; |
1313 | } |
1314 | |
1315 | void otx2_free_pending_sqe(struct otx2_nic *pfvf) |
1316 | { |
1317 | int tx_pkts = 0, tx_bytes = 0; |
1318 | struct sk_buff *skb = NULL; |
1319 | struct otx2_snd_queue *sq; |
1320 | struct netdev_queue *txq; |
1321 | struct sg_list *sg; |
1322 | int sq_idx, sqe; |
1323 | |
1324 | for (sq_idx = 0; sq_idx < pfvf->hw.tx_queues; sq_idx++) { |
1325 | sq = &pfvf->qset.sq[sq_idx]; |
1326 | for (sqe = 0; sqe < sq->sqe_cnt; sqe++) { |
1327 | sg = &sq->sg[sqe]; |
1328 | skb = (struct sk_buff *)sg->skb; |
1329 | if (skb) { |
1330 | tx_bytes += skb->len; |
1331 | tx_pkts++; |
1332 | otx2_dma_unmap_skb_frags(pfvf, sg); |
1333 | dev_kfree_skb_any(skb); |
1334 | sg->skb = (u64)NULL; |
1335 | } |
1336 | } |
1337 | |
1338 | if (!tx_pkts) |
1339 | continue; |
1340 | txq = netdev_get_tx_queue(dev: pfvf->netdev, index: sq_idx); |
1341 | netdev_tx_completed_queue(dev_queue: txq, pkts: tx_pkts, bytes: tx_bytes); |
1342 | tx_pkts = 0; |
1343 | tx_bytes = 0; |
1344 | } |
1345 | } |
1346 | |
1347 | static void otx2_xdp_sqe_add_sg(struct otx2_snd_queue *sq, u64 dma_addr, |
1348 | int len, int *offset) |
1349 | { |
1350 | struct nix_sqe_sg_s *sg = NULL; |
1351 | u64 *iova = NULL; |
1352 | |
1353 | sg = (struct nix_sqe_sg_s *)(sq->sqe_base + *offset); |
1354 | sg->ld_type = NIX_SEND_LDTYPE_LDD; |
1355 | sg->subdc = NIX_SUBDC_SG; |
1356 | sg->segs = 1; |
1357 | sg->seg1_size = len; |
1358 | iova = (void *)sg + sizeof(*sg); |
1359 | *iova = dma_addr; |
1360 | *offset += sizeof(*sg) + sizeof(u64); |
1361 | |
1362 | sq->sg[sq->head].dma_addr[0] = dma_addr; |
1363 | sq->sg[sq->head].size[0] = len; |
1364 | sq->sg[sq->head].num_segs = 1; |
1365 | } |
1366 | |
1367 | bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, u64 iova, int len, u16 qidx) |
1368 | { |
1369 | struct nix_sqe_hdr_s *sqe_hdr; |
1370 | struct otx2_snd_queue *sq; |
1371 | int offset, free_sqe; |
1372 | |
1373 | sq = &pfvf->qset.sq[qidx]; |
1374 | free_sqe = (sq->num_sqbs - *sq->aura_fc_addr) * sq->sqe_per_sqb; |
1375 | if (free_sqe < sq->sqe_thresh) |
1376 | return false; |
1377 | |
1378 | memset(sq->sqe_base + 8, 0, sq->sqe_size - 8); |
1379 | |
1380 | sqe_hdr = (struct nix_sqe_hdr_s *)(sq->sqe_base); |
1381 | |
1382 | if (!sqe_hdr->total) { |
1383 | sqe_hdr->aura = sq->aura_id; |
1384 | sqe_hdr->df = 1; |
1385 | sqe_hdr->sq = qidx; |
1386 | sqe_hdr->pnc = 1; |
1387 | } |
1388 | sqe_hdr->total = len; |
1389 | sqe_hdr->sqe_id = sq->head; |
1390 | |
1391 | offset = sizeof(*sqe_hdr); |
1392 | |
1393 | otx2_xdp_sqe_add_sg(sq, dma_addr: iova, len, offset: &offset); |
1394 | sqe_hdr->sizem1 = (offset / 16) - 1; |
1395 | pfvf->hw_ops->sqe_flush(pfvf, sq, offset, qidx); |
1396 | |
1397 | return true; |
1398 | } |
1399 | |
1400 | static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf, |
1401 | struct bpf_prog *prog, |
1402 | struct nix_cqe_rx_s *cqe, |
1403 | struct otx2_cq_queue *cq, |
1404 | bool *need_xdp_flush) |
1405 | { |
1406 | unsigned char *hard_start; |
1407 | int qidx = cq->cq_idx; |
1408 | struct xdp_buff xdp; |
1409 | struct page *page; |
1410 | u64 iova, pa; |
1411 | u32 act; |
1412 | int err; |
1413 | |
1414 | iova = cqe->sg.seg_addr - OTX2_HEAD_ROOM; |
1415 | pa = otx2_iova_to_phys(iommu_domain: pfvf->iommu_domain, dma_addr: iova); |
1416 | page = virt_to_page(phys_to_virt(pa)); |
1417 | |
1418 | xdp_init_buff(xdp: &xdp, frame_sz: pfvf->rbsize, rxq: &cq->xdp_rxq); |
1419 | |
1420 | hard_start = (unsigned char *)phys_to_virt(address: pa); |
1421 | xdp_prepare_buff(xdp: &xdp, hard_start, OTX2_HEAD_ROOM, |
1422 | data_len: cqe->sg.seg_size, meta_valid: false); |
1423 | |
1424 | act = bpf_prog_run_xdp(prog, xdp: &xdp); |
1425 | |
1426 | switch (act) { |
1427 | case XDP_PASS: |
1428 | break; |
1429 | case XDP_TX: |
1430 | qidx += pfvf->hw.tx_queues; |
1431 | cq->pool_ptrs++; |
1432 | return otx2_xdp_sq_append_pkt(pfvf, iova, |
1433 | len: cqe->sg.seg_size, qidx); |
1434 | case XDP_REDIRECT: |
1435 | cq->pool_ptrs++; |
1436 | err = xdp_do_redirect(dev: pfvf->netdev, xdp: &xdp, prog); |
1437 | |
1438 | otx2_dma_unmap_page(pfvf, addr: iova, size: pfvf->rbsize, |
1439 | dir: DMA_FROM_DEVICE); |
1440 | if (!err) { |
1441 | *need_xdp_flush = true; |
1442 | return true; |
1443 | } |
1444 | put_page(page); |
1445 | break; |
1446 | default: |
1447 | bpf_warn_invalid_xdp_action(dev: pfvf->netdev, prog, act); |
1448 | break; |
1449 | case XDP_ABORTED: |
1450 | trace_xdp_exception(dev: pfvf->netdev, xdp: prog, act); |
1451 | break; |
1452 | case XDP_DROP: |
1453 | otx2_dma_unmap_page(pfvf, addr: iova, size: pfvf->rbsize, |
1454 | dir: DMA_FROM_DEVICE); |
1455 | put_page(page); |
1456 | cq->pool_ptrs++; |
1457 | return true; |
1458 | } |
1459 | return false; |
1460 | } |
1461 | |