1/*
2 * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33#include <linux/tcp.h>
34#include <linux/if_vlan.h>
35#include <net/geneve.h>
36#include <net/dsfield.h>
37#include "en.h"
38#include "en/txrx.h"
39#include "ipoib/ipoib.h"
40#include "en_accel/en_accel.h"
41#include "en_accel/ipsec_rxtx.h"
42#include "en_accel/macsec.h"
43#include "en/ptp.h"
44#include <net/ipv6.h>
45
46static void mlx5e_dma_unmap_wqe_err(struct mlx5e_txqsq *sq, u8 num_dma)
47{
48 int i;
49
50 for (i = 0; i < num_dma; i++) {
51 struct mlx5e_sq_dma *last_pushed_dma =
52 mlx5e_dma_get(sq, i: --sq->dma_fifo_pc);
53
54 mlx5e_tx_dma_unmap(pdev: sq->pdev, dma: last_pushed_dma);
55 }
56}
57
58static inline int mlx5e_skb_l2_header_offset(struct sk_buff *skb)
59{
60#define MLX5E_MIN_INLINE (ETH_HLEN + VLAN_HLEN)
61
62 return max(skb_network_offset(skb), MLX5E_MIN_INLINE);
63}
64
65static inline int mlx5e_skb_l3_header_offset(struct sk_buff *skb)
66{
67 if (skb_transport_header_was_set(skb))
68 return skb_transport_offset(skb);
69 else
70 return mlx5e_skb_l2_header_offset(skb);
71}
72
73static inline u16 mlx5e_calc_min_inline(enum mlx5_inline_modes mode,
74 struct sk_buff *skb)
75{
76 u16 hlen;
77
78 switch (mode) {
79 case MLX5_INLINE_MODE_NONE:
80 return 0;
81 case MLX5_INLINE_MODE_TCP_UDP:
82 hlen = eth_get_headlen(dev: skb->dev, data: skb->data, len: skb_headlen(skb));
83 if (hlen == ETH_HLEN && !skb_vlan_tag_present(skb))
84 hlen += VLAN_HLEN;
85 break;
86 case MLX5_INLINE_MODE_IP:
87 hlen = mlx5e_skb_l3_header_offset(skb);
88 break;
89 case MLX5_INLINE_MODE_L2:
90 default:
91 hlen = mlx5e_skb_l2_header_offset(skb);
92 }
93 return min_t(u16, hlen, skb_headlen(skb));
94}
95
96#define MLX5_UNSAFE_MEMCPY_DISCLAIMER \
97 "This copy has been bounds-checked earlier in " \
98 "mlx5i_sq_calc_wqe_attr() and intentionally " \
99 "crosses a flex array boundary. Since it is " \
100 "performance sensitive, splitting the copy is " \
101 "undesirable."
102
103static inline void mlx5e_insert_vlan(void *start, struct sk_buff *skb, u16 ihs)
104{
105 struct vlan_ethhdr *vhdr = (struct vlan_ethhdr *)start;
106 int cpy1_sz = 2 * ETH_ALEN;
107 int cpy2_sz = ihs - cpy1_sz;
108
109 memcpy(&vhdr->addrs, skb->data, cpy1_sz);
110 vhdr->h_vlan_proto = skb->vlan_proto;
111 vhdr->h_vlan_TCI = cpu_to_be16(skb_vlan_tag_get(skb));
112 unsafe_memcpy(&vhdr->h_vlan_encapsulated_proto,
113 skb->data + cpy1_sz,
114 cpy2_sz,
115 MLX5_UNSAFE_MEMCPY_DISCLAIMER);
116}
117
118static inline void
119mlx5e_txwqe_build_eseg_csum(struct mlx5e_txqsq *sq, struct sk_buff *skb,
120 struct mlx5e_accel_tx_state *accel,
121 struct mlx5_wqe_eth_seg *eseg)
122{
123 if (unlikely(mlx5e_ipsec_txwqe_build_eseg_csum(sq, skb, eseg)))
124 return;
125
126 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
127 eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM;
128 if (skb->encapsulation) {
129 eseg->cs_flags |= MLX5_ETH_WQE_L3_INNER_CSUM |
130 MLX5_ETH_WQE_L4_INNER_CSUM;
131 sq->stats->csum_partial_inner++;
132 } else {
133 eseg->cs_flags |= MLX5_ETH_WQE_L4_CSUM;
134 sq->stats->csum_partial++;
135 }
136#ifdef CONFIG_MLX5_EN_TLS
137 } else if (unlikely(accel && accel->tls.tls_tisn)) {
138 eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM;
139 sq->stats->csum_partial++;
140#endif
141 } else
142 sq->stats->csum_none++;
143}
144
145/* Returns the number of header bytes that we plan
146 * to inline later in the transmit descriptor
147 */
148static inline u16
149mlx5e_tx_get_gso_ihs(struct mlx5e_txqsq *sq, struct sk_buff *skb, int *hopbyhop)
150{
151 struct mlx5e_sq_stats *stats = sq->stats;
152 u16 ihs;
153
154 *hopbyhop = 0;
155 if (skb->encapsulation) {
156 ihs = skb_inner_tcp_all_headers(skb);
157 stats->tso_inner_packets++;
158 stats->tso_inner_bytes += skb->len - ihs;
159 } else {
160 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
161 ihs = skb_transport_offset(skb) + sizeof(struct udphdr);
162 } else {
163 ihs = skb_tcp_all_headers(skb);
164 if (ipv6_has_hopopt_jumbo(skb)) {
165 *hopbyhop = sizeof(struct hop_jumbo_hdr);
166 ihs -= sizeof(struct hop_jumbo_hdr);
167 }
168 }
169 stats->tso_packets++;
170 stats->tso_bytes += skb->len - ihs - *hopbyhop;
171 }
172
173 return ihs;
174}
175
176static inline int
177mlx5e_txwqe_build_dsegs(struct mlx5e_txqsq *sq, struct sk_buff *skb,
178 unsigned char *skb_data, u16 headlen,
179 struct mlx5_wqe_data_seg *dseg)
180{
181 dma_addr_t dma_addr = 0;
182 u8 num_dma = 0;
183 int i;
184
185 if (headlen) {
186 dma_addr = dma_map_single(sq->pdev, skb_data, headlen,
187 DMA_TO_DEVICE);
188 if (unlikely(dma_mapping_error(sq->pdev, dma_addr)))
189 goto dma_unmap_wqe_err;
190
191 dseg->addr = cpu_to_be64(dma_addr);
192 dseg->lkey = sq->mkey_be;
193 dseg->byte_count = cpu_to_be32(headlen);
194
195 mlx5e_dma_push(sq, addr: dma_addr, size: headlen, map_type: MLX5E_DMA_MAP_SINGLE);
196 num_dma++;
197 dseg++;
198 }
199
200 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
201 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
202 int fsz = skb_frag_size(frag);
203
204 dma_addr = skb_frag_dma_map(dev: sq->pdev, frag, offset: 0, size: fsz,
205 dir: DMA_TO_DEVICE);
206 if (unlikely(dma_mapping_error(sq->pdev, dma_addr)))
207 goto dma_unmap_wqe_err;
208
209 dseg->addr = cpu_to_be64(dma_addr);
210 dseg->lkey = sq->mkey_be;
211 dseg->byte_count = cpu_to_be32(fsz);
212
213 mlx5e_dma_push(sq, addr: dma_addr, size: fsz, map_type: MLX5E_DMA_MAP_PAGE);
214 num_dma++;
215 dseg++;
216 }
217
218 return num_dma;
219
220dma_unmap_wqe_err:
221 mlx5e_dma_unmap_wqe_err(sq, num_dma);
222 return -ENOMEM;
223}
224
225struct mlx5e_tx_attr {
226 u32 num_bytes;
227 u16 headlen;
228 u16 ihs;
229 __be16 mss;
230 u16 insz;
231 u8 opcode;
232 u8 hopbyhop;
233};
234
235struct mlx5e_tx_wqe_attr {
236 u16 ds_cnt;
237 u16 ds_cnt_inl;
238 u16 ds_cnt_ids;
239 u8 num_wqebbs;
240};
241
242static u8
243mlx5e_tx_wqe_inline_mode(struct mlx5e_txqsq *sq, struct sk_buff *skb,
244 struct mlx5e_accel_tx_state *accel)
245{
246 u8 mode;
247
248#ifdef CONFIG_MLX5_EN_TLS
249 if (accel && accel->tls.tls_tisn)
250 return MLX5_INLINE_MODE_TCP_UDP;
251#endif
252
253 mode = sq->min_inline_mode;
254
255 if (skb_vlan_tag_present(skb) &&
256 test_bit(MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE, &sq->state))
257 mode = max_t(u8, MLX5_INLINE_MODE_L2, mode);
258
259 return mode;
260}
261
262static void mlx5e_sq_xmit_prepare(struct mlx5e_txqsq *sq, struct sk_buff *skb,
263 struct mlx5e_accel_tx_state *accel,
264 struct mlx5e_tx_attr *attr)
265{
266 struct mlx5e_sq_stats *stats = sq->stats;
267
268 if (skb_is_gso(skb)) {
269 int hopbyhop;
270 u16 ihs = mlx5e_tx_get_gso_ihs(sq, skb, hopbyhop: &hopbyhop);
271
272 *attr = (struct mlx5e_tx_attr) {
273 .opcode = MLX5_OPCODE_LSO,
274 .mss = cpu_to_be16(skb_shinfo(skb)->gso_size),
275 .ihs = ihs,
276 .num_bytes = skb->len + (skb_shinfo(skb)->gso_segs - 1) * ihs,
277 .headlen = skb_headlen(skb) - ihs - hopbyhop,
278 .hopbyhop = hopbyhop,
279 };
280
281 stats->packets += skb_shinfo(skb)->gso_segs;
282 } else {
283 u8 mode = mlx5e_tx_wqe_inline_mode(sq, skb, accel);
284 u16 ihs = mlx5e_calc_min_inline(mode, skb);
285
286 *attr = (struct mlx5e_tx_attr) {
287 .opcode = MLX5_OPCODE_SEND,
288 .mss = cpu_to_be16(0),
289 .ihs = ihs,
290 .num_bytes = max_t(unsigned int, skb->len, ETH_ZLEN),
291 .headlen = skb_headlen(skb) - ihs,
292 };
293
294 stats->packets++;
295 }
296
297 attr->insz = mlx5e_accel_tx_ids_len(sq, state: accel);
298 stats->bytes += attr->num_bytes;
299}
300
301static void mlx5e_sq_calc_wqe_attr(struct sk_buff *skb, const struct mlx5e_tx_attr *attr,
302 struct mlx5e_tx_wqe_attr *wqe_attr)
303{
304 u16 ds_cnt = MLX5E_TX_WQE_EMPTY_DS_COUNT;
305 u16 ds_cnt_inl = 0;
306 u16 ds_cnt_ids = 0;
307
308 /* Sync the calculation with MLX5E_MAX_TX_WQEBBS. */
309
310 if (attr->insz)
311 ds_cnt_ids = DIV_ROUND_UP(sizeof(struct mlx5_wqe_inline_seg) + attr->insz,
312 MLX5_SEND_WQE_DS);
313
314 ds_cnt += !!attr->headlen + skb_shinfo(skb)->nr_frags + ds_cnt_ids;
315 if (attr->ihs) {
316 u16 inl = attr->ihs - INL_HDR_START_SZ;
317
318 if (skb_vlan_tag_present(skb))
319 inl += VLAN_HLEN;
320
321 ds_cnt_inl = DIV_ROUND_UP(inl, MLX5_SEND_WQE_DS);
322 if (WARN_ON_ONCE(ds_cnt_inl > MLX5E_MAX_TX_INLINE_DS))
323 netdev_warn(dev: skb->dev, format: "ds_cnt_inl = %u > max %u\n", ds_cnt_inl,
324 (u16)MLX5E_MAX_TX_INLINE_DS);
325 ds_cnt += ds_cnt_inl;
326 }
327
328 *wqe_attr = (struct mlx5e_tx_wqe_attr) {
329 .ds_cnt = ds_cnt,
330 .ds_cnt_inl = ds_cnt_inl,
331 .ds_cnt_ids = ds_cnt_ids,
332 .num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS),
333 };
334}
335
336static void mlx5e_tx_skb_update_hwts_flags(struct sk_buff *skb)
337{
338 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))
339 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
340}
341
342static void mlx5e_tx_check_stop(struct mlx5e_txqsq *sq)
343{
344 if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, sq->stop_room))) {
345 netif_tx_stop_queue(dev_queue: sq->txq);
346 sq->stats->stopped++;
347 }
348}
349
350static void mlx5e_tx_flush(struct mlx5e_txqsq *sq)
351{
352 struct mlx5e_tx_wqe_info *wi;
353 struct mlx5e_tx_wqe *wqe;
354 u16 pi;
355
356 /* Must not be called when a MPWQE session is active but empty. */
357 mlx5e_tx_mpwqe_ensure_complete(sq);
358
359 pi = mlx5_wq_cyc_ctr2ix(wq: &sq->wq, ctr: sq->pc);
360 wi = &sq->db.wqe_info[pi];
361
362 *wi = (struct mlx5e_tx_wqe_info) {
363 .num_wqebbs = 1,
364 };
365
366 wqe = mlx5e_post_nop(wq: &sq->wq, sqn: sq->sqn, pc: &sq->pc);
367 mlx5e_notify_hw(wq: &sq->wq, pc: sq->pc, uar_map: sq->uar_map, ctrl: &wqe->ctrl);
368}
369
370static inline void
371mlx5e_txwqe_complete(struct mlx5e_txqsq *sq, struct sk_buff *skb,
372 const struct mlx5e_tx_attr *attr,
373 const struct mlx5e_tx_wqe_attr *wqe_attr, u8 num_dma,
374 struct mlx5e_tx_wqe_info *wi, struct mlx5_wqe_ctrl_seg *cseg,
375 struct mlx5_wqe_eth_seg *eseg, bool xmit_more)
376{
377 struct mlx5_wq_cyc *wq = &sq->wq;
378 bool send_doorbell;
379
380 *wi = (struct mlx5e_tx_wqe_info) {
381 .skb = skb,
382 .num_bytes = attr->num_bytes,
383 .num_dma = num_dma,
384 .num_wqebbs = wqe_attr->num_wqebbs,
385 .num_fifo_pkts = 0,
386 };
387
388 cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | attr->opcode);
389 cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | wqe_attr->ds_cnt);
390
391 mlx5e_tx_skb_update_hwts_flags(skb);
392
393 sq->pc += wi->num_wqebbs;
394
395 mlx5e_tx_check_stop(sq);
396
397 if (unlikely(sq->ptpsq &&
398 (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) {
399 u8 metadata_index = be32_to_cpu(eseg->flow_table_metadata);
400
401 mlx5e_ptp_metadata_fifo_pop(fifo: &sq->ptpsq->metadata_freelist);
402
403 mlx5e_skb_cb_hwtstamp_init(skb);
404 mlx5e_ptp_metadata_map_put(map: &sq->ptpsq->metadata_map, skb,
405 metadata: metadata_index);
406 /* ensure skb is put on metadata_map before tracking the index */
407 wmb();
408 mlx5e_ptpsq_track_metadata(ptpsq: sq->ptpsq, metadata: metadata_index);
409 if (!netif_tx_queue_stopped(dev_queue: sq->txq) &&
410 mlx5e_ptpsq_metadata_freelist_empty(ptpsq: sq->ptpsq)) {
411 netif_tx_stop_queue(dev_queue: sq->txq);
412 sq->stats->stopped++;
413 }
414 skb_get(skb);
415 }
416
417 send_doorbell = __netdev_tx_sent_queue(dev_queue: sq->txq, bytes: attr->num_bytes, xmit_more);
418 if (send_doorbell)
419 mlx5e_notify_hw(wq, pc: sq->pc, uar_map: sq->uar_map, ctrl: cseg);
420}
421
422static void
423mlx5e_sq_xmit_wqe(struct mlx5e_txqsq *sq, struct sk_buff *skb,
424 const struct mlx5e_tx_attr *attr, const struct mlx5e_tx_wqe_attr *wqe_attr,
425 struct mlx5e_tx_wqe *wqe, u16 pi, bool xmit_more)
426{
427 struct mlx5_wqe_ctrl_seg *cseg;
428 struct mlx5_wqe_eth_seg *eseg;
429 struct mlx5_wqe_data_seg *dseg;
430 struct mlx5e_tx_wqe_info *wi;
431 u16 ihs = attr->ihs;
432 struct ipv6hdr *h6;
433 struct mlx5e_sq_stats *stats = sq->stats;
434 int num_dma;
435
436 stats->xmit_more += xmit_more;
437
438 /* fill wqe */
439 wi = &sq->db.wqe_info[pi];
440 cseg = &wqe->ctrl;
441 eseg = &wqe->eth;
442 dseg = wqe->data;
443
444 eseg->mss = attr->mss;
445
446 if (ihs) {
447 u8 *start = eseg->inline_hdr.start;
448
449 if (unlikely(attr->hopbyhop)) {
450 /* remove the HBH header.
451 * Layout: [Ethernet header][IPv6 header][HBH][TCP header]
452 */
453 if (skb_vlan_tag_present(skb)) {
454 mlx5e_insert_vlan(start, skb, ETH_HLEN + sizeof(*h6));
455 ihs += VLAN_HLEN;
456 h6 = (struct ipv6hdr *)(start + sizeof(struct vlan_ethhdr));
457 } else {
458 unsafe_memcpy(start, skb->data,
459 ETH_HLEN + sizeof(*h6),
460 MLX5_UNSAFE_MEMCPY_DISCLAIMER);
461 h6 = (struct ipv6hdr *)(start + ETH_HLEN);
462 }
463 h6->nexthdr = IPPROTO_TCP;
464 /* Copy the TCP header after the IPv6 one */
465 memcpy(h6 + 1,
466 skb->data + ETH_HLEN + sizeof(*h6) +
467 sizeof(struct hop_jumbo_hdr),
468 tcp_hdrlen(skb));
469 /* Leave ipv6 payload_len set to 0, as LSO v2 specs request. */
470 } else if (skb_vlan_tag_present(skb)) {
471 mlx5e_insert_vlan(start, skb, ihs);
472 ihs += VLAN_HLEN;
473 stats->added_vlan_packets++;
474 } else {
475 unsafe_memcpy(eseg->inline_hdr.start, skb->data,
476 attr->ihs,
477 MLX5_UNSAFE_MEMCPY_DISCLAIMER);
478 }
479 eseg->inline_hdr.sz |= cpu_to_be16(ihs);
480 dseg += wqe_attr->ds_cnt_inl;
481 } else if (skb_vlan_tag_present(skb)) {
482 eseg->insert.type = cpu_to_be16(MLX5_ETH_WQE_INSERT_VLAN);
483 if (skb->vlan_proto == cpu_to_be16(ETH_P_8021AD))
484 eseg->insert.type |= cpu_to_be16(MLX5_ETH_WQE_SVLAN);
485 eseg->insert.vlan_tci = cpu_to_be16(skb_vlan_tag_get(skb));
486 stats->added_vlan_packets++;
487 }
488
489 dseg += wqe_attr->ds_cnt_ids;
490 num_dma = mlx5e_txwqe_build_dsegs(sq, skb, skb_data: skb->data + attr->ihs + attr->hopbyhop,
491 headlen: attr->headlen, dseg);
492 if (unlikely(num_dma < 0))
493 goto err_drop;
494
495 mlx5e_txwqe_complete(sq, skb, attr, wqe_attr, num_dma, wi, cseg, eseg, xmit_more);
496
497 return;
498
499err_drop:
500 stats->dropped++;
501 dev_kfree_skb_any(skb);
502 mlx5e_tx_flush(sq);
503}
504
505static bool mlx5e_tx_skb_supports_mpwqe(struct sk_buff *skb, struct mlx5e_tx_attr *attr)
506{
507 return !skb_is_nonlinear(skb) && !skb_vlan_tag_present(skb) && !attr->ihs &&
508 !attr->insz && !mlx5e_macsec_skb_is_offload(skb);
509}
510
511static bool mlx5e_tx_mpwqe_same_eseg(struct mlx5e_txqsq *sq, struct mlx5_wqe_eth_seg *eseg)
512{
513 struct mlx5e_tx_mpwqe *session = &sq->mpwqe;
514
515 /* Assumes the session is already running and has at least one packet. */
516 return !memcmp(p: &session->wqe->eth, q: eseg, MLX5E_ACCEL_ESEG_LEN);
517}
518
519static void mlx5e_tx_mpwqe_session_start(struct mlx5e_txqsq *sq,
520 struct mlx5_wqe_eth_seg *eseg)
521{
522 struct mlx5e_tx_mpwqe *session = &sq->mpwqe;
523 struct mlx5e_tx_wqe *wqe;
524 u16 pi;
525
526 pi = mlx5e_txqsq_get_next_pi(sq, size: sq->max_sq_mpw_wqebbs);
527 wqe = MLX5E_TX_FETCH_WQE(sq, pi);
528 net_prefetchw(p: wqe->data);
529
530 *session = (struct mlx5e_tx_mpwqe) {
531 .wqe = wqe,
532 .bytes_count = 0,
533 .ds_count = MLX5E_TX_WQE_EMPTY_DS_COUNT,
534 .pkt_count = 0,
535 .inline_on = 0,
536 };
537
538 memcpy(&session->wqe->eth, eseg, MLX5E_ACCEL_ESEG_LEN);
539
540 sq->stats->mpwqe_blks++;
541}
542
543static bool mlx5e_tx_mpwqe_session_is_active(struct mlx5e_txqsq *sq)
544{
545 return sq->mpwqe.wqe;
546}
547
548static void mlx5e_tx_mpwqe_add_dseg(struct mlx5e_txqsq *sq, struct mlx5e_xmit_data *txd)
549{
550 struct mlx5e_tx_mpwqe *session = &sq->mpwqe;
551 struct mlx5_wqe_data_seg *dseg;
552
553 dseg = (struct mlx5_wqe_data_seg *)session->wqe + session->ds_count;
554
555 session->pkt_count++;
556 session->bytes_count += txd->len;
557
558 dseg->addr = cpu_to_be64(txd->dma_addr);
559 dseg->byte_count = cpu_to_be32(txd->len);
560 dseg->lkey = sq->mkey_be;
561 session->ds_count++;
562
563 sq->stats->mpwqe_pkts++;
564}
565
566static struct mlx5_wqe_ctrl_seg *mlx5e_tx_mpwqe_session_complete(struct mlx5e_txqsq *sq)
567{
568 struct mlx5e_tx_mpwqe *session = &sq->mpwqe;
569 u8 ds_count = session->ds_count;
570 struct mlx5_wqe_ctrl_seg *cseg;
571 struct mlx5e_tx_wqe_info *wi;
572 u16 pi;
573
574 cseg = &session->wqe->ctrl;
575 cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_ENHANCED_MPSW);
576 cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_count);
577
578 pi = mlx5_wq_cyc_ctr2ix(wq: &sq->wq, ctr: sq->pc);
579 wi = &sq->db.wqe_info[pi];
580 *wi = (struct mlx5e_tx_wqe_info) {
581 .skb = NULL,
582 .num_bytes = session->bytes_count,
583 .num_wqebbs = DIV_ROUND_UP(ds_count, MLX5_SEND_WQEBB_NUM_DS),
584 .num_dma = session->pkt_count,
585 .num_fifo_pkts = session->pkt_count,
586 };
587
588 sq->pc += wi->num_wqebbs;
589
590 session->wqe = NULL;
591
592 mlx5e_tx_check_stop(sq);
593
594 return cseg;
595}
596
597static void
598mlx5e_sq_xmit_mpwqe(struct mlx5e_txqsq *sq, struct sk_buff *skb,
599 struct mlx5_wqe_eth_seg *eseg, bool xmit_more)
600{
601 struct mlx5_wqe_ctrl_seg *cseg;
602 struct mlx5e_xmit_data txd;
603
604 txd.data = skb->data;
605 txd.len = skb->len;
606
607 txd.dma_addr = dma_map_single(sq->pdev, txd.data, txd.len, DMA_TO_DEVICE);
608 if (unlikely(dma_mapping_error(sq->pdev, txd.dma_addr)))
609 goto err_unmap;
610
611 if (!mlx5e_tx_mpwqe_session_is_active(sq)) {
612 mlx5e_tx_mpwqe_session_start(sq, eseg);
613 } else if (!mlx5e_tx_mpwqe_same_eseg(sq, eseg)) {
614 mlx5e_tx_mpwqe_session_complete(sq);
615 mlx5e_tx_mpwqe_session_start(sq, eseg);
616 }
617
618 sq->stats->xmit_more += xmit_more;
619
620 mlx5e_dma_push(sq, addr: txd.dma_addr, size: txd.len, map_type: MLX5E_DMA_MAP_SINGLE);
621 mlx5e_skb_fifo_push(fifo: &sq->db.skb_fifo, skb);
622 mlx5e_tx_mpwqe_add_dseg(sq, txd: &txd);
623 mlx5e_tx_skb_update_hwts_flags(skb);
624
625 if (unlikely(mlx5e_tx_mpwqe_is_full(&sq->mpwqe, sq->max_sq_mpw_wqebbs))) {
626 /* Might stop the queue and affect the retval of __netdev_tx_sent_queue. */
627 cseg = mlx5e_tx_mpwqe_session_complete(sq);
628
629 if (__netdev_tx_sent_queue(dev_queue: sq->txq, bytes: txd.len, xmit_more))
630 mlx5e_notify_hw(wq: &sq->wq, pc: sq->pc, uar_map: sq->uar_map, ctrl: cseg);
631 } else if (__netdev_tx_sent_queue(dev_queue: sq->txq, bytes: txd.len, xmit_more)) {
632 /* Might stop the queue, but we were asked to ring the doorbell anyway. */
633 cseg = mlx5e_tx_mpwqe_session_complete(sq);
634
635 mlx5e_notify_hw(wq: &sq->wq, pc: sq->pc, uar_map: sq->uar_map, ctrl: cseg);
636 }
637
638 return;
639
640err_unmap:
641 mlx5e_dma_unmap_wqe_err(sq, num_dma: 1);
642 sq->stats->dropped++;
643 dev_kfree_skb_any(skb);
644 mlx5e_tx_flush(sq);
645}
646
647void mlx5e_tx_mpwqe_ensure_complete(struct mlx5e_txqsq *sq)
648{
649 /* Unlikely in non-MPWQE workloads; not important in MPWQE workloads. */
650 if (unlikely(mlx5e_tx_mpwqe_session_is_active(sq)))
651 mlx5e_tx_mpwqe_session_complete(sq);
652}
653
654static void mlx5e_cqe_ts_id_eseg(struct mlx5e_ptpsq *ptpsq, struct sk_buff *skb,
655 struct mlx5_wqe_eth_seg *eseg)
656{
657 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))
658 eseg->flow_table_metadata =
659 cpu_to_be32(mlx5e_ptp_metadata_fifo_peek(&ptpsq->metadata_freelist));
660}
661
662static void mlx5e_txwqe_build_eseg(struct mlx5e_priv *priv, struct mlx5e_txqsq *sq,
663 struct sk_buff *skb, struct mlx5e_accel_tx_state *accel,
664 struct mlx5_wqe_eth_seg *eseg, u16 ihs)
665{
666 mlx5e_accel_tx_eseg(priv, skb, eseg, ihs);
667 mlx5e_txwqe_build_eseg_csum(sq, skb, accel, eseg);
668 if (unlikely(sq->ptpsq))
669 mlx5e_cqe_ts_id_eseg(ptpsq: sq->ptpsq, skb, eseg);
670}
671
672netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev)
673{
674 struct mlx5e_priv *priv = netdev_priv(dev);
675 struct mlx5e_accel_tx_state accel = {};
676 struct mlx5e_tx_wqe_attr wqe_attr;
677 struct mlx5e_tx_attr attr;
678 struct mlx5e_tx_wqe *wqe;
679 struct mlx5e_txqsq *sq;
680 u16 pi;
681
682 /* All changes to txq2sq are performed in sync with mlx5e_xmit, when the
683 * queue being changed is disabled, and smp_wmb guarantees that the
684 * changes are visible before mlx5e_xmit tries to read from txq2sq. It
685 * guarantees that the value of txq2sq[qid] doesn't change while
686 * mlx5e_xmit is running on queue number qid. smb_wmb is paired with
687 * HARD_TX_LOCK around ndo_start_xmit, which serves as an ACQUIRE.
688 */
689 sq = priv->txq2sq[skb_get_queue_mapping(skb)];
690 if (unlikely(!sq)) {
691 /* Two cases when sq can be NULL:
692 * 1. The HTB node is registered, and mlx5e_select_queue
693 * selected its queue ID, but the SQ itself is not yet created.
694 * 2. HTB SQ creation failed. Similar to the previous case, but
695 * the SQ won't be created.
696 */
697 dev_kfree_skb_any(skb);
698 return NETDEV_TX_OK;
699 }
700
701 /* May send SKBs and WQEs. */
702 if (unlikely(!mlx5e_accel_tx_begin(dev, sq, skb, &accel)))
703 return NETDEV_TX_OK;
704
705 mlx5e_sq_xmit_prepare(sq, skb, accel: &accel, attr: &attr);
706
707 if (test_bit(MLX5E_SQ_STATE_MPWQE, &sq->state)) {
708 if (mlx5e_tx_skb_supports_mpwqe(skb, attr: &attr)) {
709 struct mlx5_wqe_eth_seg eseg = {};
710
711 mlx5e_txwqe_build_eseg(priv, sq, skb, accel: &accel, eseg: &eseg, ihs: attr.ihs);
712 mlx5e_sq_xmit_mpwqe(sq, skb, eseg: &eseg, xmit_more: netdev_xmit_more());
713 return NETDEV_TX_OK;
714 }
715
716 mlx5e_tx_mpwqe_ensure_complete(sq);
717 }
718
719 mlx5e_sq_calc_wqe_attr(skb, attr: &attr, wqe_attr: &wqe_attr);
720 pi = mlx5e_txqsq_get_next_pi(sq, size: wqe_attr.num_wqebbs);
721 wqe = MLX5E_TX_FETCH_WQE(sq, pi);
722
723 /* May update the WQE, but may not post other WQEs. */
724 mlx5e_accel_tx_finish(sq, wqe, state: &accel,
725 inlseg: (struct mlx5_wqe_inline_seg *)(wqe->data + wqe_attr.ds_cnt_inl));
726 mlx5e_txwqe_build_eseg(priv, sq, skb, accel: &accel, eseg: &wqe->eth, ihs: attr.ihs);
727 mlx5e_sq_xmit_wqe(sq, skb, attr: &attr, wqe_attr: &wqe_attr, wqe, pi, xmit_more: netdev_xmit_more());
728
729 return NETDEV_TX_OK;
730}
731
732static void mlx5e_tx_wi_dma_unmap(struct mlx5e_txqsq *sq, struct mlx5e_tx_wqe_info *wi,
733 u32 *dma_fifo_cc)
734{
735 int i;
736
737 for (i = 0; i < wi->num_dma; i++) {
738 struct mlx5e_sq_dma *dma = mlx5e_dma_get(sq, i: (*dma_fifo_cc)++);
739
740 mlx5e_tx_dma_unmap(pdev: sq->pdev, dma);
741 }
742}
743
744static void mlx5e_consume_skb(struct mlx5e_txqsq *sq, struct sk_buff *skb,
745 struct mlx5_cqe64 *cqe, int napi_budget)
746{
747 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
748 struct skb_shared_hwtstamps hwts = {};
749 u64 ts = get_cqe_ts(cqe);
750
751 hwts.hwtstamp = mlx5e_cqe_ts_to_ns(func: sq->ptp_cyc2time, clock: sq->clock, cqe_ts: ts);
752 if (sq->ptpsq)
753 mlx5e_skb_cb_hwtstamp_handler(skb, hwtstamp_type: MLX5E_SKB_CB_CQE_HWTSTAMP,
754 hwtstamp: hwts.hwtstamp, cq_stats: sq->ptpsq->cq_stats);
755 else
756 skb_tstamp_tx(orig_skb: skb, hwtstamps: &hwts);
757 }
758
759 napi_consume_skb(skb, budget: napi_budget);
760}
761
762static void mlx5e_tx_wi_consume_fifo_skbs(struct mlx5e_txqsq *sq, struct mlx5e_tx_wqe_info *wi,
763 struct mlx5_cqe64 *cqe, int napi_budget)
764{
765 int i;
766
767 for (i = 0; i < wi->num_fifo_pkts; i++) {
768 struct sk_buff *skb = mlx5e_skb_fifo_pop(fifo: &sq->db.skb_fifo);
769
770 mlx5e_consume_skb(sq, skb, cqe, napi_budget);
771 }
772}
773
774void mlx5e_txqsq_wake(struct mlx5e_txqsq *sq)
775{
776 if (netif_tx_queue_stopped(dev_queue: sq->txq) &&
777 mlx5e_wqc_has_room_for(wq: &sq->wq, cc: sq->cc, pc: sq->pc, n: sq->stop_room) &&
778 !mlx5e_ptpsq_metadata_freelist_empty(ptpsq: sq->ptpsq) &&
779 !test_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state)) {
780 netif_tx_wake_queue(dev_queue: sq->txq);
781 sq->stats->wake++;
782 }
783}
784
785bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget)
786{
787 struct mlx5e_sq_stats *stats;
788 struct mlx5e_txqsq *sq;
789 struct mlx5_cqe64 *cqe;
790 u32 dma_fifo_cc;
791 u32 nbytes;
792 u16 npkts;
793 u16 sqcc;
794 int i;
795
796 sq = container_of(cq, struct mlx5e_txqsq, cq);
797
798 if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
799 return false;
800
801 cqe = mlx5_cqwq_get_cqe(wq: &cq->wq);
802 if (!cqe)
803 return false;
804
805 stats = sq->stats;
806
807 npkts = 0;
808 nbytes = 0;
809
810 /* sq->cc must be updated only after mlx5_cqwq_update_db_record(),
811 * otherwise a cq overrun may occur
812 */
813 sqcc = sq->cc;
814
815 /* avoid dirtying sq cache line every cqe */
816 dma_fifo_cc = sq->dma_fifo_cc;
817
818 i = 0;
819 do {
820 struct mlx5e_tx_wqe_info *wi;
821 u16 wqe_counter;
822 bool last_wqe;
823 u16 ci;
824
825 mlx5_cqwq_pop(wq: &cq->wq);
826
827 wqe_counter = be16_to_cpu(cqe->wqe_counter);
828
829 do {
830 last_wqe = (sqcc == wqe_counter);
831
832 ci = mlx5_wq_cyc_ctr2ix(wq: &sq->wq, ctr: sqcc);
833 wi = &sq->db.wqe_info[ci];
834
835 sqcc += wi->num_wqebbs;
836
837 if (likely(wi->skb)) {
838 mlx5e_tx_wi_dma_unmap(sq, wi, dma_fifo_cc: &dma_fifo_cc);
839 mlx5e_consume_skb(sq, skb: wi->skb, cqe, napi_budget);
840
841 npkts++;
842 nbytes += wi->num_bytes;
843 continue;
844 }
845
846 if (unlikely(mlx5e_ktls_tx_try_handle_resync_dump_comp(sq, wi,
847 &dma_fifo_cc)))
848 continue;
849
850 if (wi->num_fifo_pkts) {
851 mlx5e_tx_wi_dma_unmap(sq, wi, dma_fifo_cc: &dma_fifo_cc);
852 mlx5e_tx_wi_consume_fifo_skbs(sq, wi, cqe, napi_budget);
853
854 npkts += wi->num_fifo_pkts;
855 nbytes += wi->num_bytes;
856 }
857 } while (!last_wqe);
858
859 if (unlikely(get_cqe_opcode(cqe) == MLX5_CQE_REQ_ERR)) {
860 if (!test_and_set_bit(nr: MLX5E_SQ_STATE_RECOVERING,
861 addr: &sq->state)) {
862 mlx5e_dump_error_cqe(cq: &sq->cq, qn: sq->sqn,
863 err_cqe: (struct mlx5_err_cqe *)cqe);
864 mlx5_wq_cyc_wqe_dump(wq: &sq->wq, ix: ci, nstrides: wi->num_wqebbs);
865 queue_work(wq: cq->workqueue, work: &sq->recover_work);
866 }
867 stats->cqe_err++;
868 }
869
870 } while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(wq: &cq->wq)));
871
872 stats->cqes += i;
873
874 mlx5_cqwq_update_db_record(wq: &cq->wq);
875
876 /* ensure cq space is freed before enabling more cqes */
877 wmb();
878
879 sq->dma_fifo_cc = dma_fifo_cc;
880 sq->cc = sqcc;
881
882 netdev_tx_completed_queue(dev_queue: sq->txq, pkts: npkts, bytes: nbytes);
883
884 mlx5e_txqsq_wake(sq);
885
886 return (i == MLX5E_TX_CQ_POLL_BUDGET);
887}
888
889static void mlx5e_tx_wi_kfree_fifo_skbs(struct mlx5e_txqsq *sq, struct mlx5e_tx_wqe_info *wi)
890{
891 int i;
892
893 for (i = 0; i < wi->num_fifo_pkts; i++)
894 dev_kfree_skb_any(skb: mlx5e_skb_fifo_pop(fifo: &sq->db.skb_fifo));
895}
896
897void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq)
898{
899 struct mlx5e_tx_wqe_info *wi;
900 u32 dma_fifo_cc, nbytes = 0;
901 u16 ci, sqcc, npkts = 0;
902
903 sqcc = sq->cc;
904 dma_fifo_cc = sq->dma_fifo_cc;
905
906 while (sqcc != sq->pc) {
907 ci = mlx5_wq_cyc_ctr2ix(wq: &sq->wq, ctr: sqcc);
908 wi = &sq->db.wqe_info[ci];
909
910 sqcc += wi->num_wqebbs;
911
912 if (likely(wi->skb)) {
913 mlx5e_tx_wi_dma_unmap(sq, wi, dma_fifo_cc: &dma_fifo_cc);
914 dev_kfree_skb_any(skb: wi->skb);
915
916 npkts++;
917 nbytes += wi->num_bytes;
918 continue;
919 }
920
921 if (unlikely(mlx5e_ktls_tx_try_handle_resync_dump_comp(sq, wi, &dma_fifo_cc)))
922 continue;
923
924 if (wi->num_fifo_pkts) {
925 mlx5e_tx_wi_dma_unmap(sq, wi, dma_fifo_cc: &dma_fifo_cc);
926 mlx5e_tx_wi_kfree_fifo_skbs(sq, wi);
927
928 npkts += wi->num_fifo_pkts;
929 nbytes += wi->num_bytes;
930 }
931 }
932
933 sq->dma_fifo_cc = dma_fifo_cc;
934 sq->cc = sqcc;
935
936 netdev_tx_completed_queue(dev_queue: sq->txq, pkts: npkts, bytes: nbytes);
937}
938
939#ifdef CONFIG_MLX5_CORE_IPOIB
940static inline void
941mlx5i_txwqe_build_datagram(struct mlx5_av *av, u32 dqpn, u32 dqkey,
942 struct mlx5_wqe_datagram_seg *dseg)
943{
944 memcpy(&dseg->av, av, sizeof(struct mlx5_av));
945 dseg->av.dqp_dct = cpu_to_be32(dqpn | MLX5_EXTENDED_UD_AV);
946 dseg->av.key.qkey.qkey = cpu_to_be32(dqkey);
947}
948
949static void mlx5i_sq_calc_wqe_attr(struct sk_buff *skb,
950 const struct mlx5e_tx_attr *attr,
951 struct mlx5e_tx_wqe_attr *wqe_attr)
952{
953 u16 ds_cnt = sizeof(struct mlx5i_tx_wqe) / MLX5_SEND_WQE_DS;
954 u16 ds_cnt_inl = 0;
955
956 ds_cnt += !!attr->headlen + skb_shinfo(skb)->nr_frags;
957
958 if (attr->ihs) {
959 u16 inl = attr->ihs - INL_HDR_START_SZ;
960
961 ds_cnt_inl = DIV_ROUND_UP(inl, MLX5_SEND_WQE_DS);
962 ds_cnt += ds_cnt_inl;
963 }
964
965 *wqe_attr = (struct mlx5e_tx_wqe_attr) {
966 .ds_cnt = ds_cnt,
967 .ds_cnt_inl = ds_cnt_inl,
968 .num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS),
969 };
970}
971
972void mlx5i_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb,
973 struct mlx5_av *av, u32 dqpn, u32 dqkey, bool xmit_more)
974{
975 struct mlx5e_tx_wqe_attr wqe_attr;
976 struct mlx5e_tx_attr attr;
977 struct mlx5i_tx_wqe *wqe;
978
979 struct mlx5_wqe_datagram_seg *datagram;
980 struct mlx5_wqe_ctrl_seg *cseg;
981 struct mlx5_wqe_eth_seg *eseg;
982 struct mlx5_wqe_data_seg *dseg;
983 struct mlx5e_tx_wqe_info *wi;
984
985 struct mlx5e_sq_stats *stats = sq->stats;
986 int num_dma;
987 u16 pi;
988
989 mlx5e_sq_xmit_prepare(sq, skb, NULL, attr: &attr);
990 mlx5i_sq_calc_wqe_attr(skb, attr: &attr, wqe_attr: &wqe_attr);
991
992 pi = mlx5e_txqsq_get_next_pi(sq, size: wqe_attr.num_wqebbs);
993 wqe = MLX5I_SQ_FETCH_WQE(sq, pi);
994
995 stats->xmit_more += xmit_more;
996
997 /* fill wqe */
998 wi = &sq->db.wqe_info[pi];
999 cseg = &wqe->ctrl;
1000 datagram = &wqe->datagram;
1001 eseg = &wqe->eth;
1002 dseg = wqe->data;
1003
1004 mlx5i_txwqe_build_datagram(av, dqpn, dqkey, dseg: datagram);
1005
1006 mlx5e_txwqe_build_eseg_csum(sq, skb, NULL, eseg);
1007
1008 eseg->mss = attr.mss;
1009
1010 if (attr.ihs) {
1011 if (unlikely(attr.hopbyhop)) {
1012 struct ipv6hdr *h6;
1013
1014 /* remove the HBH header.
1015 * Layout: [Ethernet header][IPv6 header][HBH][TCP header]
1016 */
1017 unsafe_memcpy(eseg->inline_hdr.start, skb->data,
1018 ETH_HLEN + sizeof(*h6),
1019 MLX5_UNSAFE_MEMCPY_DISCLAIMER);
1020 h6 = (struct ipv6hdr *)((char *)eseg->inline_hdr.start + ETH_HLEN);
1021 h6->nexthdr = IPPROTO_TCP;
1022 /* Copy the TCP header after the IPv6 one */
1023 unsafe_memcpy(h6 + 1,
1024 skb->data + ETH_HLEN + sizeof(*h6) +
1025 sizeof(struct hop_jumbo_hdr),
1026 tcp_hdrlen(skb),
1027 MLX5_UNSAFE_MEMCPY_DISCLAIMER);
1028 /* Leave ipv6 payload_len set to 0, as LSO v2 specs request. */
1029 } else {
1030 unsafe_memcpy(eseg->inline_hdr.start, skb->data,
1031 attr.ihs,
1032 MLX5_UNSAFE_MEMCPY_DISCLAIMER);
1033 }
1034 eseg->inline_hdr.sz = cpu_to_be16(attr.ihs);
1035 dseg += wqe_attr.ds_cnt_inl;
1036 }
1037
1038 num_dma = mlx5e_txwqe_build_dsegs(sq, skb, skb_data: skb->data + attr.ihs + attr.hopbyhop,
1039 headlen: attr.headlen, dseg);
1040 if (unlikely(num_dma < 0))
1041 goto err_drop;
1042
1043 mlx5e_txwqe_complete(sq, skb, attr: &attr, wqe_attr: &wqe_attr, num_dma, wi, cseg, eseg, xmit_more);
1044
1045 return;
1046
1047err_drop:
1048 stats->dropped++;
1049 dev_kfree_skb_any(skb);
1050 mlx5e_tx_flush(sq);
1051}
1052#endif
1053

source code of linux/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c