1 | /* |
2 | * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved. |
3 | * |
4 | * This software is available to you under a choice of one of two |
5 | * licenses. You may choose to be licensed under the terms of the GNU |
6 | * General Public License (GPL) Version 2, available from the file |
7 | * COPYING in the main directory of this source tree, or the |
8 | * OpenIB.org BSD license below: |
9 | * |
10 | * Redistribution and use in source and binary forms, with or |
11 | * without modification, are permitted provided that the following |
12 | * conditions are met: |
13 | * |
14 | * - Redistributions of source code must retain the above |
15 | * copyright notice, this list of conditions and the following |
16 | * disclaimer. |
17 | * |
18 | * - Redistributions in binary form must reproduce the above |
19 | * copyright notice, this list of conditions and the following |
20 | * disclaimer in the documentation and/or other materials |
21 | * provided with the distribution. |
22 | * |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
30 | * SOFTWARE. |
31 | */ |
32 | |
33 | #include <linux/tcp.h> |
34 | #include <linux/if_vlan.h> |
35 | #include <net/geneve.h> |
36 | #include <net/dsfield.h> |
37 | #include "en.h" |
38 | #include "en/txrx.h" |
39 | #include "ipoib/ipoib.h" |
40 | #include "en_accel/en_accel.h" |
41 | #include "en_accel/ipsec_rxtx.h" |
42 | #include "en_accel/macsec.h" |
43 | #include "en/ptp.h" |
44 | #include <net/ipv6.h> |
45 | |
46 | static void mlx5e_dma_unmap_wqe_err(struct mlx5e_txqsq *sq, u8 num_dma) |
47 | { |
48 | int i; |
49 | |
50 | for (i = 0; i < num_dma; i++) { |
51 | struct mlx5e_sq_dma *last_pushed_dma = |
52 | mlx5e_dma_get(sq, i: --sq->dma_fifo_pc); |
53 | |
54 | mlx5e_tx_dma_unmap(pdev: sq->pdev, dma: last_pushed_dma); |
55 | } |
56 | } |
57 | |
58 | static inline int (struct sk_buff *skb) |
59 | { |
60 | #define MLX5E_MIN_INLINE (ETH_HLEN + VLAN_HLEN) |
61 | |
62 | return max(skb_network_offset(skb), MLX5E_MIN_INLINE); |
63 | } |
64 | |
65 | static inline int (struct sk_buff *skb) |
66 | { |
67 | if (skb_transport_header_was_set(skb)) |
68 | return skb_transport_offset(skb); |
69 | else |
70 | return mlx5e_skb_l2_header_offset(skb); |
71 | } |
72 | |
73 | static inline u16 mlx5e_calc_min_inline(enum mlx5_inline_modes mode, |
74 | struct sk_buff *skb) |
75 | { |
76 | u16 hlen; |
77 | |
78 | switch (mode) { |
79 | case MLX5_INLINE_MODE_NONE: |
80 | return 0; |
81 | case MLX5_INLINE_MODE_TCP_UDP: |
82 | hlen = eth_get_headlen(dev: skb->dev, data: skb->data, len: skb_headlen(skb)); |
83 | if (hlen == ETH_HLEN && !skb_vlan_tag_present(skb)) |
84 | hlen += VLAN_HLEN; |
85 | break; |
86 | case MLX5_INLINE_MODE_IP: |
87 | hlen = mlx5e_skb_l3_header_offset(skb); |
88 | break; |
89 | case MLX5_INLINE_MODE_L2: |
90 | default: |
91 | hlen = mlx5e_skb_l2_header_offset(skb); |
92 | } |
93 | return min_t(u16, hlen, skb_headlen(skb)); |
94 | } |
95 | |
96 | #define MLX5_UNSAFE_MEMCPY_DISCLAIMER \ |
97 | "This copy has been bounds-checked earlier in " \ |
98 | "mlx5i_sq_calc_wqe_attr() and intentionally " \ |
99 | "crosses a flex array boundary. Since it is " \ |
100 | "performance sensitive, splitting the copy is " \ |
101 | "undesirable." |
102 | |
103 | static inline void mlx5e_insert_vlan(void *start, struct sk_buff *skb, u16 ihs) |
104 | { |
105 | struct vlan_ethhdr *vhdr = (struct vlan_ethhdr *)start; |
106 | int cpy1_sz = 2 * ETH_ALEN; |
107 | int cpy2_sz = ihs - cpy1_sz; |
108 | |
109 | memcpy(&vhdr->addrs, skb->data, cpy1_sz); |
110 | vhdr->h_vlan_proto = skb->vlan_proto; |
111 | vhdr->h_vlan_TCI = cpu_to_be16(skb_vlan_tag_get(skb)); |
112 | unsafe_memcpy(&vhdr->h_vlan_encapsulated_proto, |
113 | skb->data + cpy1_sz, |
114 | cpy2_sz, |
115 | MLX5_UNSAFE_MEMCPY_DISCLAIMER); |
116 | } |
117 | |
118 | static inline void |
119 | mlx5e_txwqe_build_eseg_csum(struct mlx5e_txqsq *sq, struct sk_buff *skb, |
120 | struct mlx5e_accel_tx_state *accel, |
121 | struct mlx5_wqe_eth_seg *eseg) |
122 | { |
123 | if (unlikely(mlx5e_ipsec_txwqe_build_eseg_csum(sq, skb, eseg))) |
124 | return; |
125 | |
126 | if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { |
127 | eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM; |
128 | if (skb->encapsulation) { |
129 | eseg->cs_flags |= MLX5_ETH_WQE_L3_INNER_CSUM | |
130 | MLX5_ETH_WQE_L4_INNER_CSUM; |
131 | sq->stats->csum_partial_inner++; |
132 | } else { |
133 | eseg->cs_flags |= MLX5_ETH_WQE_L4_CSUM; |
134 | sq->stats->csum_partial++; |
135 | } |
136 | #ifdef CONFIG_MLX5_EN_TLS |
137 | } else if (unlikely(accel && accel->tls.tls_tisn)) { |
138 | eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM; |
139 | sq->stats->csum_partial++; |
140 | #endif |
141 | } else |
142 | sq->stats->csum_none++; |
143 | } |
144 | |
145 | /* Returns the number of header bytes that we plan |
146 | * to inline later in the transmit descriptor |
147 | */ |
148 | static inline u16 |
149 | mlx5e_tx_get_gso_ihs(struct mlx5e_txqsq *sq, struct sk_buff *skb, int *hopbyhop) |
150 | { |
151 | struct mlx5e_sq_stats *stats = sq->stats; |
152 | u16 ihs; |
153 | |
154 | *hopbyhop = 0; |
155 | if (skb->encapsulation) { |
156 | ihs = skb_inner_tcp_all_headers(skb); |
157 | stats->tso_inner_packets++; |
158 | stats->tso_inner_bytes += skb->len - ihs; |
159 | } else { |
160 | if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { |
161 | ihs = skb_transport_offset(skb) + sizeof(struct udphdr); |
162 | } else { |
163 | ihs = skb_tcp_all_headers(skb); |
164 | if (ipv6_has_hopopt_jumbo(skb)) { |
165 | *hopbyhop = sizeof(struct hop_jumbo_hdr); |
166 | ihs -= sizeof(struct hop_jumbo_hdr); |
167 | } |
168 | } |
169 | stats->tso_packets++; |
170 | stats->tso_bytes += skb->len - ihs - *hopbyhop; |
171 | } |
172 | |
173 | return ihs; |
174 | } |
175 | |
176 | static inline int |
177 | mlx5e_txwqe_build_dsegs(struct mlx5e_txqsq *sq, struct sk_buff *skb, |
178 | unsigned char *skb_data, u16 headlen, |
179 | struct mlx5_wqe_data_seg *dseg) |
180 | { |
181 | dma_addr_t dma_addr = 0; |
182 | u8 num_dma = 0; |
183 | int i; |
184 | |
185 | if (headlen) { |
186 | dma_addr = dma_map_single(sq->pdev, skb_data, headlen, |
187 | DMA_TO_DEVICE); |
188 | if (unlikely(dma_mapping_error(sq->pdev, dma_addr))) |
189 | goto dma_unmap_wqe_err; |
190 | |
191 | dseg->addr = cpu_to_be64(dma_addr); |
192 | dseg->lkey = sq->mkey_be; |
193 | dseg->byte_count = cpu_to_be32(headlen); |
194 | |
195 | mlx5e_dma_push(sq, addr: dma_addr, size: headlen, map_type: MLX5E_DMA_MAP_SINGLE); |
196 | num_dma++; |
197 | dseg++; |
198 | } |
199 | |
200 | for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { |
201 | skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; |
202 | int fsz = skb_frag_size(frag); |
203 | |
204 | dma_addr = skb_frag_dma_map(dev: sq->pdev, frag, offset: 0, size: fsz, |
205 | dir: DMA_TO_DEVICE); |
206 | if (unlikely(dma_mapping_error(sq->pdev, dma_addr))) |
207 | goto dma_unmap_wqe_err; |
208 | |
209 | dseg->addr = cpu_to_be64(dma_addr); |
210 | dseg->lkey = sq->mkey_be; |
211 | dseg->byte_count = cpu_to_be32(fsz); |
212 | |
213 | mlx5e_dma_push(sq, addr: dma_addr, size: fsz, map_type: MLX5E_DMA_MAP_PAGE); |
214 | num_dma++; |
215 | dseg++; |
216 | } |
217 | |
218 | return num_dma; |
219 | |
220 | dma_unmap_wqe_err: |
221 | mlx5e_dma_unmap_wqe_err(sq, num_dma); |
222 | return -ENOMEM; |
223 | } |
224 | |
225 | struct mlx5e_tx_attr { |
226 | u32 num_bytes; |
227 | u16 headlen; |
228 | u16 ihs; |
229 | __be16 mss; |
230 | u16 insz; |
231 | u8 opcode; |
232 | u8 hopbyhop; |
233 | }; |
234 | |
235 | struct mlx5e_tx_wqe_attr { |
236 | u16 ds_cnt; |
237 | u16 ds_cnt_inl; |
238 | u16 ds_cnt_ids; |
239 | u8 num_wqebbs; |
240 | }; |
241 | |
242 | static u8 |
243 | mlx5e_tx_wqe_inline_mode(struct mlx5e_txqsq *sq, struct sk_buff *skb, |
244 | struct mlx5e_accel_tx_state *accel) |
245 | { |
246 | u8 mode; |
247 | |
248 | #ifdef CONFIG_MLX5_EN_TLS |
249 | if (accel && accel->tls.tls_tisn) |
250 | return MLX5_INLINE_MODE_TCP_UDP; |
251 | #endif |
252 | |
253 | mode = sq->min_inline_mode; |
254 | |
255 | if (skb_vlan_tag_present(skb) && |
256 | test_bit(MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE, &sq->state)) |
257 | mode = max_t(u8, MLX5_INLINE_MODE_L2, mode); |
258 | |
259 | return mode; |
260 | } |
261 | |
262 | static void mlx5e_sq_xmit_prepare(struct mlx5e_txqsq *sq, struct sk_buff *skb, |
263 | struct mlx5e_accel_tx_state *accel, |
264 | struct mlx5e_tx_attr *attr) |
265 | { |
266 | struct mlx5e_sq_stats *stats = sq->stats; |
267 | |
268 | if (skb_is_gso(skb)) { |
269 | int hopbyhop; |
270 | u16 ihs = mlx5e_tx_get_gso_ihs(sq, skb, hopbyhop: &hopbyhop); |
271 | |
272 | *attr = (struct mlx5e_tx_attr) { |
273 | .opcode = MLX5_OPCODE_LSO, |
274 | .mss = cpu_to_be16(skb_shinfo(skb)->gso_size), |
275 | .ihs = ihs, |
276 | .num_bytes = skb->len + (skb_shinfo(skb)->gso_segs - 1) * ihs, |
277 | .headlen = skb_headlen(skb) - ihs - hopbyhop, |
278 | .hopbyhop = hopbyhop, |
279 | }; |
280 | |
281 | stats->packets += skb_shinfo(skb)->gso_segs; |
282 | } else { |
283 | u8 mode = mlx5e_tx_wqe_inline_mode(sq, skb, accel); |
284 | u16 ihs = mlx5e_calc_min_inline(mode, skb); |
285 | |
286 | *attr = (struct mlx5e_tx_attr) { |
287 | .opcode = MLX5_OPCODE_SEND, |
288 | .mss = cpu_to_be16(0), |
289 | .ihs = ihs, |
290 | .num_bytes = max_t(unsigned int, skb->len, ETH_ZLEN), |
291 | .headlen = skb_headlen(skb) - ihs, |
292 | }; |
293 | |
294 | stats->packets++; |
295 | } |
296 | |
297 | attr->insz = mlx5e_accel_tx_ids_len(sq, state: accel); |
298 | stats->bytes += attr->num_bytes; |
299 | } |
300 | |
301 | static void mlx5e_sq_calc_wqe_attr(struct sk_buff *skb, const struct mlx5e_tx_attr *attr, |
302 | struct mlx5e_tx_wqe_attr *wqe_attr) |
303 | { |
304 | u16 ds_cnt = MLX5E_TX_WQE_EMPTY_DS_COUNT; |
305 | u16 ds_cnt_inl = 0; |
306 | u16 ds_cnt_ids = 0; |
307 | |
308 | /* Sync the calculation with MLX5E_MAX_TX_WQEBBS. */ |
309 | |
310 | if (attr->insz) |
311 | ds_cnt_ids = DIV_ROUND_UP(sizeof(struct mlx5_wqe_inline_seg) + attr->insz, |
312 | MLX5_SEND_WQE_DS); |
313 | |
314 | ds_cnt += !!attr->headlen + skb_shinfo(skb)->nr_frags + ds_cnt_ids; |
315 | if (attr->ihs) { |
316 | u16 inl = attr->ihs - INL_HDR_START_SZ; |
317 | |
318 | if (skb_vlan_tag_present(skb)) |
319 | inl += VLAN_HLEN; |
320 | |
321 | ds_cnt_inl = DIV_ROUND_UP(inl, MLX5_SEND_WQE_DS); |
322 | if (WARN_ON_ONCE(ds_cnt_inl > MLX5E_MAX_TX_INLINE_DS)) |
323 | netdev_warn(dev: skb->dev, format: "ds_cnt_inl = %u > max %u\n" , ds_cnt_inl, |
324 | (u16)MLX5E_MAX_TX_INLINE_DS); |
325 | ds_cnt += ds_cnt_inl; |
326 | } |
327 | |
328 | *wqe_attr = (struct mlx5e_tx_wqe_attr) { |
329 | .ds_cnt = ds_cnt, |
330 | .ds_cnt_inl = ds_cnt_inl, |
331 | .ds_cnt_ids = ds_cnt_ids, |
332 | .num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS), |
333 | }; |
334 | } |
335 | |
336 | static void mlx5e_tx_skb_update_hwts_flags(struct sk_buff *skb) |
337 | { |
338 | if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) |
339 | skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; |
340 | } |
341 | |
342 | static void mlx5e_tx_check_stop(struct mlx5e_txqsq *sq) |
343 | { |
344 | if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, sq->stop_room))) { |
345 | netif_tx_stop_queue(dev_queue: sq->txq); |
346 | sq->stats->stopped++; |
347 | } |
348 | } |
349 | |
350 | static void mlx5e_tx_flush(struct mlx5e_txqsq *sq) |
351 | { |
352 | struct mlx5e_tx_wqe_info *wi; |
353 | struct mlx5e_tx_wqe *wqe; |
354 | u16 pi; |
355 | |
356 | /* Must not be called when a MPWQE session is active but empty. */ |
357 | mlx5e_tx_mpwqe_ensure_complete(sq); |
358 | |
359 | pi = mlx5_wq_cyc_ctr2ix(wq: &sq->wq, ctr: sq->pc); |
360 | wi = &sq->db.wqe_info[pi]; |
361 | |
362 | *wi = (struct mlx5e_tx_wqe_info) { |
363 | .num_wqebbs = 1, |
364 | }; |
365 | |
366 | wqe = mlx5e_post_nop(wq: &sq->wq, sqn: sq->sqn, pc: &sq->pc); |
367 | mlx5e_notify_hw(wq: &sq->wq, pc: sq->pc, uar_map: sq->uar_map, ctrl: &wqe->ctrl); |
368 | } |
369 | |
370 | static inline void |
371 | mlx5e_txwqe_complete(struct mlx5e_txqsq *sq, struct sk_buff *skb, |
372 | const struct mlx5e_tx_attr *attr, |
373 | const struct mlx5e_tx_wqe_attr *wqe_attr, u8 num_dma, |
374 | struct mlx5e_tx_wqe_info *wi, struct mlx5_wqe_ctrl_seg *cseg, |
375 | struct mlx5_wqe_eth_seg *eseg, bool xmit_more) |
376 | { |
377 | struct mlx5_wq_cyc *wq = &sq->wq; |
378 | bool send_doorbell; |
379 | |
380 | *wi = (struct mlx5e_tx_wqe_info) { |
381 | .skb = skb, |
382 | .num_bytes = attr->num_bytes, |
383 | .num_dma = num_dma, |
384 | .num_wqebbs = wqe_attr->num_wqebbs, |
385 | .num_fifo_pkts = 0, |
386 | }; |
387 | |
388 | cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | attr->opcode); |
389 | cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | wqe_attr->ds_cnt); |
390 | |
391 | mlx5e_tx_skb_update_hwts_flags(skb); |
392 | |
393 | sq->pc += wi->num_wqebbs; |
394 | |
395 | mlx5e_tx_check_stop(sq); |
396 | |
397 | if (unlikely(sq->ptpsq && |
398 | (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) { |
399 | u8 metadata_index = be32_to_cpu(eseg->flow_table_metadata); |
400 | |
401 | mlx5e_ptp_metadata_fifo_pop(fifo: &sq->ptpsq->metadata_freelist); |
402 | |
403 | mlx5e_skb_cb_hwtstamp_init(skb); |
404 | mlx5e_ptp_metadata_map_put(map: &sq->ptpsq->metadata_map, skb, |
405 | metadata: metadata_index); |
406 | /* ensure skb is put on metadata_map before tracking the index */ |
407 | wmb(); |
408 | mlx5e_ptpsq_track_metadata(ptpsq: sq->ptpsq, metadata: metadata_index); |
409 | if (!netif_tx_queue_stopped(dev_queue: sq->txq) && |
410 | mlx5e_ptpsq_metadata_freelist_empty(ptpsq: sq->ptpsq)) { |
411 | netif_tx_stop_queue(dev_queue: sq->txq); |
412 | sq->stats->stopped++; |
413 | } |
414 | skb_get(skb); |
415 | } |
416 | |
417 | send_doorbell = __netdev_tx_sent_queue(dev_queue: sq->txq, bytes: attr->num_bytes, xmit_more); |
418 | if (send_doorbell) |
419 | mlx5e_notify_hw(wq, pc: sq->pc, uar_map: sq->uar_map, ctrl: cseg); |
420 | } |
421 | |
422 | static void |
423 | mlx5e_sq_xmit_wqe(struct mlx5e_txqsq *sq, struct sk_buff *skb, |
424 | const struct mlx5e_tx_attr *attr, const struct mlx5e_tx_wqe_attr *wqe_attr, |
425 | struct mlx5e_tx_wqe *wqe, u16 pi, bool xmit_more) |
426 | { |
427 | struct mlx5_wqe_ctrl_seg *cseg; |
428 | struct mlx5_wqe_eth_seg *eseg; |
429 | struct mlx5_wqe_data_seg *dseg; |
430 | struct mlx5e_tx_wqe_info *wi; |
431 | u16 ihs = attr->ihs; |
432 | struct ipv6hdr *h6; |
433 | struct mlx5e_sq_stats *stats = sq->stats; |
434 | int num_dma; |
435 | |
436 | stats->xmit_more += xmit_more; |
437 | |
438 | /* fill wqe */ |
439 | wi = &sq->db.wqe_info[pi]; |
440 | cseg = &wqe->ctrl; |
441 | eseg = &wqe->eth; |
442 | dseg = wqe->data; |
443 | |
444 | eseg->mss = attr->mss; |
445 | |
446 | if (ihs) { |
447 | u8 *start = eseg->inline_hdr.start; |
448 | |
449 | if (unlikely(attr->hopbyhop)) { |
450 | /* remove the HBH header. |
451 | * Layout: [Ethernet header][IPv6 header][HBH][TCP header] |
452 | */ |
453 | if (skb_vlan_tag_present(skb)) { |
454 | mlx5e_insert_vlan(start, skb, ETH_HLEN + sizeof(*h6)); |
455 | ihs += VLAN_HLEN; |
456 | h6 = (struct ipv6hdr *)(start + sizeof(struct vlan_ethhdr)); |
457 | } else { |
458 | unsafe_memcpy(start, skb->data, |
459 | ETH_HLEN + sizeof(*h6), |
460 | MLX5_UNSAFE_MEMCPY_DISCLAIMER); |
461 | h6 = (struct ipv6hdr *)(start + ETH_HLEN); |
462 | } |
463 | h6->nexthdr = IPPROTO_TCP; |
464 | /* Copy the TCP header after the IPv6 one */ |
465 | memcpy(h6 + 1, |
466 | skb->data + ETH_HLEN + sizeof(*h6) + |
467 | sizeof(struct hop_jumbo_hdr), |
468 | tcp_hdrlen(skb)); |
469 | /* Leave ipv6 payload_len set to 0, as LSO v2 specs request. */ |
470 | } else if (skb_vlan_tag_present(skb)) { |
471 | mlx5e_insert_vlan(start, skb, ihs); |
472 | ihs += VLAN_HLEN; |
473 | stats->added_vlan_packets++; |
474 | } else { |
475 | unsafe_memcpy(eseg->inline_hdr.start, skb->data, |
476 | attr->ihs, |
477 | MLX5_UNSAFE_MEMCPY_DISCLAIMER); |
478 | } |
479 | eseg->inline_hdr.sz |= cpu_to_be16(ihs); |
480 | dseg += wqe_attr->ds_cnt_inl; |
481 | } else if (skb_vlan_tag_present(skb)) { |
482 | eseg->insert.type = cpu_to_be16(MLX5_ETH_WQE_INSERT_VLAN); |
483 | if (skb->vlan_proto == cpu_to_be16(ETH_P_8021AD)) |
484 | eseg->insert.type |= cpu_to_be16(MLX5_ETH_WQE_SVLAN); |
485 | eseg->insert.vlan_tci = cpu_to_be16(skb_vlan_tag_get(skb)); |
486 | stats->added_vlan_packets++; |
487 | } |
488 | |
489 | dseg += wqe_attr->ds_cnt_ids; |
490 | num_dma = mlx5e_txwqe_build_dsegs(sq, skb, skb_data: skb->data + attr->ihs + attr->hopbyhop, |
491 | headlen: attr->headlen, dseg); |
492 | if (unlikely(num_dma < 0)) |
493 | goto err_drop; |
494 | |
495 | mlx5e_txwqe_complete(sq, skb, attr, wqe_attr, num_dma, wi, cseg, eseg, xmit_more); |
496 | |
497 | return; |
498 | |
499 | err_drop: |
500 | stats->dropped++; |
501 | dev_kfree_skb_any(skb); |
502 | mlx5e_tx_flush(sq); |
503 | } |
504 | |
505 | static bool mlx5e_tx_skb_supports_mpwqe(struct sk_buff *skb, struct mlx5e_tx_attr *attr) |
506 | { |
507 | return !skb_is_nonlinear(skb) && !skb_vlan_tag_present(skb) && !attr->ihs && |
508 | !attr->insz && !mlx5e_macsec_skb_is_offload(skb); |
509 | } |
510 | |
511 | static bool mlx5e_tx_mpwqe_same_eseg(struct mlx5e_txqsq *sq, struct mlx5_wqe_eth_seg *eseg) |
512 | { |
513 | struct mlx5e_tx_mpwqe *session = &sq->mpwqe; |
514 | |
515 | /* Assumes the session is already running and has at least one packet. */ |
516 | return !memcmp(p: &session->wqe->eth, q: eseg, MLX5E_ACCEL_ESEG_LEN); |
517 | } |
518 | |
519 | static void mlx5e_tx_mpwqe_session_start(struct mlx5e_txqsq *sq, |
520 | struct mlx5_wqe_eth_seg *eseg) |
521 | { |
522 | struct mlx5e_tx_mpwqe *session = &sq->mpwqe; |
523 | struct mlx5e_tx_wqe *wqe; |
524 | u16 pi; |
525 | |
526 | pi = mlx5e_txqsq_get_next_pi(sq, size: sq->max_sq_mpw_wqebbs); |
527 | wqe = MLX5E_TX_FETCH_WQE(sq, pi); |
528 | net_prefetchw(p: wqe->data); |
529 | |
530 | *session = (struct mlx5e_tx_mpwqe) { |
531 | .wqe = wqe, |
532 | .bytes_count = 0, |
533 | .ds_count = MLX5E_TX_WQE_EMPTY_DS_COUNT, |
534 | .pkt_count = 0, |
535 | .inline_on = 0, |
536 | }; |
537 | |
538 | memcpy(&session->wqe->eth, eseg, MLX5E_ACCEL_ESEG_LEN); |
539 | |
540 | sq->stats->mpwqe_blks++; |
541 | } |
542 | |
543 | static bool mlx5e_tx_mpwqe_session_is_active(struct mlx5e_txqsq *sq) |
544 | { |
545 | return sq->mpwqe.wqe; |
546 | } |
547 | |
548 | static void mlx5e_tx_mpwqe_add_dseg(struct mlx5e_txqsq *sq, struct mlx5e_xmit_data *txd) |
549 | { |
550 | struct mlx5e_tx_mpwqe *session = &sq->mpwqe; |
551 | struct mlx5_wqe_data_seg *dseg; |
552 | |
553 | dseg = (struct mlx5_wqe_data_seg *)session->wqe + session->ds_count; |
554 | |
555 | session->pkt_count++; |
556 | session->bytes_count += txd->len; |
557 | |
558 | dseg->addr = cpu_to_be64(txd->dma_addr); |
559 | dseg->byte_count = cpu_to_be32(txd->len); |
560 | dseg->lkey = sq->mkey_be; |
561 | session->ds_count++; |
562 | |
563 | sq->stats->mpwqe_pkts++; |
564 | } |
565 | |
566 | static struct mlx5_wqe_ctrl_seg *mlx5e_tx_mpwqe_session_complete(struct mlx5e_txqsq *sq) |
567 | { |
568 | struct mlx5e_tx_mpwqe *session = &sq->mpwqe; |
569 | u8 ds_count = session->ds_count; |
570 | struct mlx5_wqe_ctrl_seg *cseg; |
571 | struct mlx5e_tx_wqe_info *wi; |
572 | u16 pi; |
573 | |
574 | cseg = &session->wqe->ctrl; |
575 | cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_ENHANCED_MPSW); |
576 | cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_count); |
577 | |
578 | pi = mlx5_wq_cyc_ctr2ix(wq: &sq->wq, ctr: sq->pc); |
579 | wi = &sq->db.wqe_info[pi]; |
580 | *wi = (struct mlx5e_tx_wqe_info) { |
581 | .skb = NULL, |
582 | .num_bytes = session->bytes_count, |
583 | .num_wqebbs = DIV_ROUND_UP(ds_count, MLX5_SEND_WQEBB_NUM_DS), |
584 | .num_dma = session->pkt_count, |
585 | .num_fifo_pkts = session->pkt_count, |
586 | }; |
587 | |
588 | sq->pc += wi->num_wqebbs; |
589 | |
590 | session->wqe = NULL; |
591 | |
592 | mlx5e_tx_check_stop(sq); |
593 | |
594 | return cseg; |
595 | } |
596 | |
597 | static void |
598 | mlx5e_sq_xmit_mpwqe(struct mlx5e_txqsq *sq, struct sk_buff *skb, |
599 | struct mlx5_wqe_eth_seg *eseg, bool xmit_more) |
600 | { |
601 | struct mlx5_wqe_ctrl_seg *cseg; |
602 | struct mlx5e_xmit_data txd; |
603 | |
604 | txd.data = skb->data; |
605 | txd.len = skb->len; |
606 | |
607 | txd.dma_addr = dma_map_single(sq->pdev, txd.data, txd.len, DMA_TO_DEVICE); |
608 | if (unlikely(dma_mapping_error(sq->pdev, txd.dma_addr))) |
609 | goto err_unmap; |
610 | |
611 | if (!mlx5e_tx_mpwqe_session_is_active(sq)) { |
612 | mlx5e_tx_mpwqe_session_start(sq, eseg); |
613 | } else if (!mlx5e_tx_mpwqe_same_eseg(sq, eseg)) { |
614 | mlx5e_tx_mpwqe_session_complete(sq); |
615 | mlx5e_tx_mpwqe_session_start(sq, eseg); |
616 | } |
617 | |
618 | sq->stats->xmit_more += xmit_more; |
619 | |
620 | mlx5e_dma_push(sq, addr: txd.dma_addr, size: txd.len, map_type: MLX5E_DMA_MAP_SINGLE); |
621 | mlx5e_skb_fifo_push(fifo: &sq->db.skb_fifo, skb); |
622 | mlx5e_tx_mpwqe_add_dseg(sq, txd: &txd); |
623 | mlx5e_tx_skb_update_hwts_flags(skb); |
624 | |
625 | if (unlikely(mlx5e_tx_mpwqe_is_full(&sq->mpwqe, sq->max_sq_mpw_wqebbs))) { |
626 | /* Might stop the queue and affect the retval of __netdev_tx_sent_queue. */ |
627 | cseg = mlx5e_tx_mpwqe_session_complete(sq); |
628 | |
629 | if (__netdev_tx_sent_queue(dev_queue: sq->txq, bytes: txd.len, xmit_more)) |
630 | mlx5e_notify_hw(wq: &sq->wq, pc: sq->pc, uar_map: sq->uar_map, ctrl: cseg); |
631 | } else if (__netdev_tx_sent_queue(dev_queue: sq->txq, bytes: txd.len, xmit_more)) { |
632 | /* Might stop the queue, but we were asked to ring the doorbell anyway. */ |
633 | cseg = mlx5e_tx_mpwqe_session_complete(sq); |
634 | |
635 | mlx5e_notify_hw(wq: &sq->wq, pc: sq->pc, uar_map: sq->uar_map, ctrl: cseg); |
636 | } |
637 | |
638 | return; |
639 | |
640 | err_unmap: |
641 | mlx5e_dma_unmap_wqe_err(sq, num_dma: 1); |
642 | sq->stats->dropped++; |
643 | dev_kfree_skb_any(skb); |
644 | mlx5e_tx_flush(sq); |
645 | } |
646 | |
647 | void mlx5e_tx_mpwqe_ensure_complete(struct mlx5e_txqsq *sq) |
648 | { |
649 | /* Unlikely in non-MPWQE workloads; not important in MPWQE workloads. */ |
650 | if (unlikely(mlx5e_tx_mpwqe_session_is_active(sq))) |
651 | mlx5e_tx_mpwqe_session_complete(sq); |
652 | } |
653 | |
654 | static void mlx5e_cqe_ts_id_eseg(struct mlx5e_ptpsq *ptpsq, struct sk_buff *skb, |
655 | struct mlx5_wqe_eth_seg *eseg) |
656 | { |
657 | if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) |
658 | eseg->flow_table_metadata = |
659 | cpu_to_be32(mlx5e_ptp_metadata_fifo_peek(&ptpsq->metadata_freelist)); |
660 | } |
661 | |
662 | static void mlx5e_txwqe_build_eseg(struct mlx5e_priv *priv, struct mlx5e_txqsq *sq, |
663 | struct sk_buff *skb, struct mlx5e_accel_tx_state *accel, |
664 | struct mlx5_wqe_eth_seg *eseg, u16 ihs) |
665 | { |
666 | mlx5e_accel_tx_eseg(priv, skb, eseg, ihs); |
667 | mlx5e_txwqe_build_eseg_csum(sq, skb, accel, eseg); |
668 | if (unlikely(sq->ptpsq)) |
669 | mlx5e_cqe_ts_id_eseg(ptpsq: sq->ptpsq, skb, eseg); |
670 | } |
671 | |
672 | netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev) |
673 | { |
674 | struct mlx5e_priv *priv = netdev_priv(dev); |
675 | struct mlx5e_accel_tx_state accel = {}; |
676 | struct mlx5e_tx_wqe_attr wqe_attr; |
677 | struct mlx5e_tx_attr attr; |
678 | struct mlx5e_tx_wqe *wqe; |
679 | struct mlx5e_txqsq *sq; |
680 | u16 pi; |
681 | |
682 | /* All changes to txq2sq are performed in sync with mlx5e_xmit, when the |
683 | * queue being changed is disabled, and smp_wmb guarantees that the |
684 | * changes are visible before mlx5e_xmit tries to read from txq2sq. It |
685 | * guarantees that the value of txq2sq[qid] doesn't change while |
686 | * mlx5e_xmit is running on queue number qid. smb_wmb is paired with |
687 | * HARD_TX_LOCK around ndo_start_xmit, which serves as an ACQUIRE. |
688 | */ |
689 | sq = priv->txq2sq[skb_get_queue_mapping(skb)]; |
690 | if (unlikely(!sq)) { |
691 | /* Two cases when sq can be NULL: |
692 | * 1. The HTB node is registered, and mlx5e_select_queue |
693 | * selected its queue ID, but the SQ itself is not yet created. |
694 | * 2. HTB SQ creation failed. Similar to the previous case, but |
695 | * the SQ won't be created. |
696 | */ |
697 | dev_kfree_skb_any(skb); |
698 | return NETDEV_TX_OK; |
699 | } |
700 | |
701 | /* May send SKBs and WQEs. */ |
702 | if (unlikely(!mlx5e_accel_tx_begin(dev, sq, skb, &accel))) |
703 | return NETDEV_TX_OK; |
704 | |
705 | mlx5e_sq_xmit_prepare(sq, skb, accel: &accel, attr: &attr); |
706 | |
707 | if (test_bit(MLX5E_SQ_STATE_MPWQE, &sq->state)) { |
708 | if (mlx5e_tx_skb_supports_mpwqe(skb, attr: &attr)) { |
709 | struct mlx5_wqe_eth_seg eseg = {}; |
710 | |
711 | mlx5e_txwqe_build_eseg(priv, sq, skb, accel: &accel, eseg: &eseg, ihs: attr.ihs); |
712 | mlx5e_sq_xmit_mpwqe(sq, skb, eseg: &eseg, xmit_more: netdev_xmit_more()); |
713 | return NETDEV_TX_OK; |
714 | } |
715 | |
716 | mlx5e_tx_mpwqe_ensure_complete(sq); |
717 | } |
718 | |
719 | mlx5e_sq_calc_wqe_attr(skb, attr: &attr, wqe_attr: &wqe_attr); |
720 | pi = mlx5e_txqsq_get_next_pi(sq, size: wqe_attr.num_wqebbs); |
721 | wqe = MLX5E_TX_FETCH_WQE(sq, pi); |
722 | |
723 | /* May update the WQE, but may not post other WQEs. */ |
724 | mlx5e_accel_tx_finish(sq, wqe, state: &accel, |
725 | inlseg: (struct mlx5_wqe_inline_seg *)(wqe->data + wqe_attr.ds_cnt_inl)); |
726 | mlx5e_txwqe_build_eseg(priv, sq, skb, accel: &accel, eseg: &wqe->eth, ihs: attr.ihs); |
727 | mlx5e_sq_xmit_wqe(sq, skb, attr: &attr, wqe_attr: &wqe_attr, wqe, pi, xmit_more: netdev_xmit_more()); |
728 | |
729 | return NETDEV_TX_OK; |
730 | } |
731 | |
732 | static void mlx5e_tx_wi_dma_unmap(struct mlx5e_txqsq *sq, struct mlx5e_tx_wqe_info *wi, |
733 | u32 *dma_fifo_cc) |
734 | { |
735 | int i; |
736 | |
737 | for (i = 0; i < wi->num_dma; i++) { |
738 | struct mlx5e_sq_dma *dma = mlx5e_dma_get(sq, i: (*dma_fifo_cc)++); |
739 | |
740 | mlx5e_tx_dma_unmap(pdev: sq->pdev, dma); |
741 | } |
742 | } |
743 | |
744 | static void mlx5e_consume_skb(struct mlx5e_txqsq *sq, struct sk_buff *skb, |
745 | struct mlx5_cqe64 *cqe, int napi_budget) |
746 | { |
747 | if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { |
748 | struct skb_shared_hwtstamps hwts = {}; |
749 | u64 ts = get_cqe_ts(cqe); |
750 | |
751 | hwts.hwtstamp = mlx5e_cqe_ts_to_ns(func: sq->ptp_cyc2time, clock: sq->clock, cqe_ts: ts); |
752 | if (sq->ptpsq) |
753 | mlx5e_skb_cb_hwtstamp_handler(skb, hwtstamp_type: MLX5E_SKB_CB_CQE_HWTSTAMP, |
754 | hwtstamp: hwts.hwtstamp, cq_stats: sq->ptpsq->cq_stats); |
755 | else |
756 | skb_tstamp_tx(orig_skb: skb, hwtstamps: &hwts); |
757 | } |
758 | |
759 | napi_consume_skb(skb, budget: napi_budget); |
760 | } |
761 | |
762 | static void mlx5e_tx_wi_consume_fifo_skbs(struct mlx5e_txqsq *sq, struct mlx5e_tx_wqe_info *wi, |
763 | struct mlx5_cqe64 *cqe, int napi_budget) |
764 | { |
765 | int i; |
766 | |
767 | for (i = 0; i < wi->num_fifo_pkts; i++) { |
768 | struct sk_buff *skb = mlx5e_skb_fifo_pop(fifo: &sq->db.skb_fifo); |
769 | |
770 | mlx5e_consume_skb(sq, skb, cqe, napi_budget); |
771 | } |
772 | } |
773 | |
774 | void mlx5e_txqsq_wake(struct mlx5e_txqsq *sq) |
775 | { |
776 | if (netif_tx_queue_stopped(dev_queue: sq->txq) && |
777 | mlx5e_wqc_has_room_for(wq: &sq->wq, cc: sq->cc, pc: sq->pc, n: sq->stop_room) && |
778 | !mlx5e_ptpsq_metadata_freelist_empty(ptpsq: sq->ptpsq) && |
779 | !test_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state)) { |
780 | netif_tx_wake_queue(dev_queue: sq->txq); |
781 | sq->stats->wake++; |
782 | } |
783 | } |
784 | |
785 | bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget) |
786 | { |
787 | struct mlx5e_sq_stats *stats; |
788 | struct mlx5e_txqsq *sq; |
789 | struct mlx5_cqe64 *cqe; |
790 | u32 dma_fifo_cc; |
791 | u32 nbytes; |
792 | u16 npkts; |
793 | u16 sqcc; |
794 | int i; |
795 | |
796 | sq = container_of(cq, struct mlx5e_txqsq, cq); |
797 | |
798 | if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state))) |
799 | return false; |
800 | |
801 | cqe = mlx5_cqwq_get_cqe(wq: &cq->wq); |
802 | if (!cqe) |
803 | return false; |
804 | |
805 | stats = sq->stats; |
806 | |
807 | npkts = 0; |
808 | nbytes = 0; |
809 | |
810 | /* sq->cc must be updated only after mlx5_cqwq_update_db_record(), |
811 | * otherwise a cq overrun may occur |
812 | */ |
813 | sqcc = sq->cc; |
814 | |
815 | /* avoid dirtying sq cache line every cqe */ |
816 | dma_fifo_cc = sq->dma_fifo_cc; |
817 | |
818 | i = 0; |
819 | do { |
820 | struct mlx5e_tx_wqe_info *wi; |
821 | u16 wqe_counter; |
822 | bool last_wqe; |
823 | u16 ci; |
824 | |
825 | mlx5_cqwq_pop(wq: &cq->wq); |
826 | |
827 | wqe_counter = be16_to_cpu(cqe->wqe_counter); |
828 | |
829 | do { |
830 | last_wqe = (sqcc == wqe_counter); |
831 | |
832 | ci = mlx5_wq_cyc_ctr2ix(wq: &sq->wq, ctr: sqcc); |
833 | wi = &sq->db.wqe_info[ci]; |
834 | |
835 | sqcc += wi->num_wqebbs; |
836 | |
837 | if (likely(wi->skb)) { |
838 | mlx5e_tx_wi_dma_unmap(sq, wi, dma_fifo_cc: &dma_fifo_cc); |
839 | mlx5e_consume_skb(sq, skb: wi->skb, cqe, napi_budget); |
840 | |
841 | npkts++; |
842 | nbytes += wi->num_bytes; |
843 | continue; |
844 | } |
845 | |
846 | if (unlikely(mlx5e_ktls_tx_try_handle_resync_dump_comp(sq, wi, |
847 | &dma_fifo_cc))) |
848 | continue; |
849 | |
850 | if (wi->num_fifo_pkts) { |
851 | mlx5e_tx_wi_dma_unmap(sq, wi, dma_fifo_cc: &dma_fifo_cc); |
852 | mlx5e_tx_wi_consume_fifo_skbs(sq, wi, cqe, napi_budget); |
853 | |
854 | npkts += wi->num_fifo_pkts; |
855 | nbytes += wi->num_bytes; |
856 | } |
857 | } while (!last_wqe); |
858 | |
859 | if (unlikely(get_cqe_opcode(cqe) == MLX5_CQE_REQ_ERR)) { |
860 | if (!test_and_set_bit(nr: MLX5E_SQ_STATE_RECOVERING, |
861 | addr: &sq->state)) { |
862 | mlx5e_dump_error_cqe(cq: &sq->cq, qn: sq->sqn, |
863 | err_cqe: (struct mlx5_err_cqe *)cqe); |
864 | mlx5_wq_cyc_wqe_dump(wq: &sq->wq, ix: ci, nstrides: wi->num_wqebbs); |
865 | queue_work(wq: cq->workqueue, work: &sq->recover_work); |
866 | } |
867 | stats->cqe_err++; |
868 | } |
869 | |
870 | } while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(wq: &cq->wq))); |
871 | |
872 | stats->cqes += i; |
873 | |
874 | mlx5_cqwq_update_db_record(wq: &cq->wq); |
875 | |
876 | /* ensure cq space is freed before enabling more cqes */ |
877 | wmb(); |
878 | |
879 | sq->dma_fifo_cc = dma_fifo_cc; |
880 | sq->cc = sqcc; |
881 | |
882 | netdev_tx_completed_queue(dev_queue: sq->txq, pkts: npkts, bytes: nbytes); |
883 | |
884 | mlx5e_txqsq_wake(sq); |
885 | |
886 | return (i == MLX5E_TX_CQ_POLL_BUDGET); |
887 | } |
888 | |
889 | static void mlx5e_tx_wi_kfree_fifo_skbs(struct mlx5e_txqsq *sq, struct mlx5e_tx_wqe_info *wi) |
890 | { |
891 | int i; |
892 | |
893 | for (i = 0; i < wi->num_fifo_pkts; i++) |
894 | dev_kfree_skb_any(skb: mlx5e_skb_fifo_pop(fifo: &sq->db.skb_fifo)); |
895 | } |
896 | |
897 | void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq) |
898 | { |
899 | struct mlx5e_tx_wqe_info *wi; |
900 | u32 dma_fifo_cc, nbytes = 0; |
901 | u16 ci, sqcc, npkts = 0; |
902 | |
903 | sqcc = sq->cc; |
904 | dma_fifo_cc = sq->dma_fifo_cc; |
905 | |
906 | while (sqcc != sq->pc) { |
907 | ci = mlx5_wq_cyc_ctr2ix(wq: &sq->wq, ctr: sqcc); |
908 | wi = &sq->db.wqe_info[ci]; |
909 | |
910 | sqcc += wi->num_wqebbs; |
911 | |
912 | if (likely(wi->skb)) { |
913 | mlx5e_tx_wi_dma_unmap(sq, wi, dma_fifo_cc: &dma_fifo_cc); |
914 | dev_kfree_skb_any(skb: wi->skb); |
915 | |
916 | npkts++; |
917 | nbytes += wi->num_bytes; |
918 | continue; |
919 | } |
920 | |
921 | if (unlikely(mlx5e_ktls_tx_try_handle_resync_dump_comp(sq, wi, &dma_fifo_cc))) |
922 | continue; |
923 | |
924 | if (wi->num_fifo_pkts) { |
925 | mlx5e_tx_wi_dma_unmap(sq, wi, dma_fifo_cc: &dma_fifo_cc); |
926 | mlx5e_tx_wi_kfree_fifo_skbs(sq, wi); |
927 | |
928 | npkts += wi->num_fifo_pkts; |
929 | nbytes += wi->num_bytes; |
930 | } |
931 | } |
932 | |
933 | sq->dma_fifo_cc = dma_fifo_cc; |
934 | sq->cc = sqcc; |
935 | |
936 | netdev_tx_completed_queue(dev_queue: sq->txq, pkts: npkts, bytes: nbytes); |
937 | } |
938 | |
939 | #ifdef CONFIG_MLX5_CORE_IPOIB |
940 | static inline void |
941 | mlx5i_txwqe_build_datagram(struct mlx5_av *av, u32 dqpn, u32 dqkey, |
942 | struct mlx5_wqe_datagram_seg *dseg) |
943 | { |
944 | memcpy(&dseg->av, av, sizeof(struct mlx5_av)); |
945 | dseg->av.dqp_dct = cpu_to_be32(dqpn | MLX5_EXTENDED_UD_AV); |
946 | dseg->av.key.qkey.qkey = cpu_to_be32(dqkey); |
947 | } |
948 | |
949 | static void mlx5i_sq_calc_wqe_attr(struct sk_buff *skb, |
950 | const struct mlx5e_tx_attr *attr, |
951 | struct mlx5e_tx_wqe_attr *wqe_attr) |
952 | { |
953 | u16 ds_cnt = sizeof(struct mlx5i_tx_wqe) / MLX5_SEND_WQE_DS; |
954 | u16 ds_cnt_inl = 0; |
955 | |
956 | ds_cnt += !!attr->headlen + skb_shinfo(skb)->nr_frags; |
957 | |
958 | if (attr->ihs) { |
959 | u16 inl = attr->ihs - INL_HDR_START_SZ; |
960 | |
961 | ds_cnt_inl = DIV_ROUND_UP(inl, MLX5_SEND_WQE_DS); |
962 | ds_cnt += ds_cnt_inl; |
963 | } |
964 | |
965 | *wqe_attr = (struct mlx5e_tx_wqe_attr) { |
966 | .ds_cnt = ds_cnt, |
967 | .ds_cnt_inl = ds_cnt_inl, |
968 | .num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS), |
969 | }; |
970 | } |
971 | |
972 | void mlx5i_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb, |
973 | struct mlx5_av *av, u32 dqpn, u32 dqkey, bool xmit_more) |
974 | { |
975 | struct mlx5e_tx_wqe_attr wqe_attr; |
976 | struct mlx5e_tx_attr attr; |
977 | struct mlx5i_tx_wqe *wqe; |
978 | |
979 | struct mlx5_wqe_datagram_seg *datagram; |
980 | struct mlx5_wqe_ctrl_seg *cseg; |
981 | struct mlx5_wqe_eth_seg *eseg; |
982 | struct mlx5_wqe_data_seg *dseg; |
983 | struct mlx5e_tx_wqe_info *wi; |
984 | |
985 | struct mlx5e_sq_stats *stats = sq->stats; |
986 | int num_dma; |
987 | u16 pi; |
988 | |
989 | mlx5e_sq_xmit_prepare(sq, skb, NULL, attr: &attr); |
990 | mlx5i_sq_calc_wqe_attr(skb, attr: &attr, wqe_attr: &wqe_attr); |
991 | |
992 | pi = mlx5e_txqsq_get_next_pi(sq, size: wqe_attr.num_wqebbs); |
993 | wqe = MLX5I_SQ_FETCH_WQE(sq, pi); |
994 | |
995 | stats->xmit_more += xmit_more; |
996 | |
997 | /* fill wqe */ |
998 | wi = &sq->db.wqe_info[pi]; |
999 | cseg = &wqe->ctrl; |
1000 | datagram = &wqe->datagram; |
1001 | eseg = &wqe->eth; |
1002 | dseg = wqe->data; |
1003 | |
1004 | mlx5i_txwqe_build_datagram(av, dqpn, dqkey, dseg: datagram); |
1005 | |
1006 | mlx5e_txwqe_build_eseg_csum(sq, skb, NULL, eseg); |
1007 | |
1008 | eseg->mss = attr.mss; |
1009 | |
1010 | if (attr.ihs) { |
1011 | if (unlikely(attr.hopbyhop)) { |
1012 | struct ipv6hdr *h6; |
1013 | |
1014 | /* remove the HBH header. |
1015 | * Layout: [Ethernet header][IPv6 header][HBH][TCP header] |
1016 | */ |
1017 | unsafe_memcpy(eseg->inline_hdr.start, skb->data, |
1018 | ETH_HLEN + sizeof(*h6), |
1019 | MLX5_UNSAFE_MEMCPY_DISCLAIMER); |
1020 | h6 = (struct ipv6hdr *)((char *)eseg->inline_hdr.start + ETH_HLEN); |
1021 | h6->nexthdr = IPPROTO_TCP; |
1022 | /* Copy the TCP header after the IPv6 one */ |
1023 | unsafe_memcpy(h6 + 1, |
1024 | skb->data + ETH_HLEN + sizeof(*h6) + |
1025 | sizeof(struct hop_jumbo_hdr), |
1026 | tcp_hdrlen(skb), |
1027 | MLX5_UNSAFE_MEMCPY_DISCLAIMER); |
1028 | /* Leave ipv6 payload_len set to 0, as LSO v2 specs request. */ |
1029 | } else { |
1030 | unsafe_memcpy(eseg->inline_hdr.start, skb->data, |
1031 | attr.ihs, |
1032 | MLX5_UNSAFE_MEMCPY_DISCLAIMER); |
1033 | } |
1034 | eseg->inline_hdr.sz = cpu_to_be16(attr.ihs); |
1035 | dseg += wqe_attr.ds_cnt_inl; |
1036 | } |
1037 | |
1038 | num_dma = mlx5e_txwqe_build_dsegs(sq, skb, skb_data: skb->data + attr.ihs + attr.hopbyhop, |
1039 | headlen: attr.headlen, dseg); |
1040 | if (unlikely(num_dma < 0)) |
1041 | goto err_drop; |
1042 | |
1043 | mlx5e_txwqe_complete(sq, skb, attr: &attr, wqe_attr: &wqe_attr, num_dma, wi, cseg, eseg, xmit_more); |
1044 | |
1045 | return; |
1046 | |
1047 | err_drop: |
1048 | stats->dropped++; |
1049 | dev_kfree_skb_any(skb); |
1050 | mlx5e_tx_flush(sq); |
1051 | } |
1052 | #endif |
1053 | |