1 | /* QLogic qedr NIC Driver |
2 | * Copyright (c) 2015-2016 QLogic Corporation |
3 | * |
4 | * This software is available to you under a choice of one of two |
5 | * licenses. You may choose to be licensed under the terms of the GNU |
6 | * General Public License (GPL) Version 2, available from the file |
7 | * COPYING in the main directory of this source tree, or the |
8 | * OpenIB.org BSD license below: |
9 | * |
10 | * Redistribution and use in source and binary forms, with or |
11 | * without modification, are permitted provided that the following |
12 | * conditions are met: |
13 | * |
14 | * - Redistributions of source code must retain the above |
15 | * copyright notice, this list of conditions and the following |
16 | * disclaimer. |
17 | * |
18 | * - Redistributions in binary form must reproduce the above |
19 | * copyright notice, this list of conditions and the following |
20 | * disclaimer in the documentation and /or other materials |
21 | * provided with the distribution. |
22 | * |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
30 | * SOFTWARE. |
31 | */ |
32 | #include <linux/dma-mapping.h> |
33 | #include <linux/crc32.h> |
34 | #include <linux/iommu.h> |
35 | #include <net/ip.h> |
36 | #include <net/ipv6.h> |
37 | #include <net/udp.h> |
38 | |
39 | #include <rdma/ib_verbs.h> |
40 | #include <rdma/ib_user_verbs.h> |
41 | #include <rdma/iw_cm.h> |
42 | #include <rdma/ib_umem.h> |
43 | #include <rdma/ib_addr.h> |
44 | #include <rdma/ib_cache.h> |
45 | |
46 | #include <linux/qed/qed_if.h> |
47 | #include <linux/qed/qed_rdma_if.h> |
48 | #include "qedr.h" |
49 | #include "verbs.h" |
50 | #include <rdma/qedr-abi.h> |
51 | #include "qedr_roce_cm.h" |
52 | |
53 | void qedr_inc_sw_gsi_cons(struct qedr_qp_hwq_info *info) |
54 | { |
55 | info->gsi_cons = (info->gsi_cons + 1) % info->max_wr; |
56 | } |
57 | |
58 | void qedr_store_gsi_qp_cq(struct qedr_dev *dev, struct qedr_qp *qp, |
59 | struct ib_qp_init_attr *attrs) |
60 | { |
61 | dev->gsi_qp_created = 1; |
62 | dev->gsi_sqcq = get_qedr_cq(ibcq: attrs->send_cq); |
63 | dev->gsi_rqcq = get_qedr_cq(ibcq: attrs->recv_cq); |
64 | dev->gsi_qp = qp; |
65 | } |
66 | |
67 | static void qedr_ll2_complete_tx_packet(void *cxt, u8 connection_handle, |
68 | void *cookie, |
69 | dma_addr_t first_frag_addr, |
70 | bool b_last_fragment, |
71 | bool b_last_packet) |
72 | { |
73 | struct qedr_dev *dev = (struct qedr_dev *)cxt; |
74 | struct qed_roce_ll2_packet *pkt = cookie; |
75 | struct qedr_cq *cq = dev->gsi_sqcq; |
76 | struct qedr_qp *qp = dev->gsi_qp; |
77 | unsigned long flags; |
78 | |
79 | DP_DEBUG(dev, QEDR_MSG_GSI, |
80 | "LL2 TX CB: gsi_sqcq=%p, gsi_rqcq=%p, gsi_cons=%d, ibcq_comp=%s\n" , |
81 | dev->gsi_sqcq, dev->gsi_rqcq, qp->sq.gsi_cons, |
82 | cq->ibcq.comp_handler ? "Yes" : "No" ); |
83 | |
84 | dma_free_coherent(dev: &dev->pdev->dev, size: pkt->header.len, cpu_addr: pkt->header.vaddr, |
85 | dma_handle: pkt->header.baddr); |
86 | kfree(objp: pkt); |
87 | |
88 | spin_lock_irqsave(&qp->q_lock, flags); |
89 | qedr_inc_sw_gsi_cons(info: &qp->sq); |
90 | spin_unlock_irqrestore(lock: &qp->q_lock, flags); |
91 | |
92 | if (cq->ibcq.comp_handler) |
93 | (*cq->ibcq.comp_handler) (&cq->ibcq, cq->ibcq.cq_context); |
94 | } |
95 | |
96 | static void qedr_ll2_complete_rx_packet(void *cxt, |
97 | struct qed_ll2_comp_rx_data *data) |
98 | { |
99 | struct qedr_dev *dev = (struct qedr_dev *)cxt; |
100 | struct qedr_cq *cq = dev->gsi_rqcq; |
101 | struct qedr_qp *qp = dev->gsi_qp; |
102 | unsigned long flags; |
103 | |
104 | spin_lock_irqsave(&qp->q_lock, flags); |
105 | |
106 | qp->rqe_wr_id[qp->rq.gsi_cons].rc = data->u.data_length_error ? |
107 | -EINVAL : 0; |
108 | qp->rqe_wr_id[qp->rq.gsi_cons].vlan = data->vlan; |
109 | /* note: length stands for data length i.e. GRH is excluded */ |
110 | qp->rqe_wr_id[qp->rq.gsi_cons].sg_list[0].length = |
111 | data->length.data_length; |
112 | *((u32 *)&qp->rqe_wr_id[qp->rq.gsi_cons].smac[0]) = |
113 | ntohl(data->opaque_data_0); |
114 | *((u16 *)&qp->rqe_wr_id[qp->rq.gsi_cons].smac[4]) = |
115 | ntohs((u16)data->opaque_data_1); |
116 | |
117 | qedr_inc_sw_gsi_cons(info: &qp->rq); |
118 | |
119 | spin_unlock_irqrestore(lock: &qp->q_lock, flags); |
120 | |
121 | if (cq->ibcq.comp_handler) |
122 | (*cq->ibcq.comp_handler) (&cq->ibcq, cq->ibcq.cq_context); |
123 | } |
124 | |
125 | static void qedr_ll2_release_rx_packet(void *cxt, u8 connection_handle, |
126 | void *cookie, dma_addr_t rx_buf_addr, |
127 | bool b_last_packet) |
128 | { |
129 | /* Do nothing... */ |
130 | } |
131 | |
132 | static void qedr_destroy_gsi_cq(struct qedr_dev *dev, |
133 | struct ib_qp_init_attr *attrs) |
134 | { |
135 | struct qed_rdma_destroy_cq_in_params iparams; |
136 | struct qed_rdma_destroy_cq_out_params oparams; |
137 | struct qedr_cq *cq; |
138 | |
139 | cq = get_qedr_cq(ibcq: attrs->send_cq); |
140 | iparams.icid = cq->icid; |
141 | dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); |
142 | dev->ops->common->chain_free(dev->cdev, &cq->pbl); |
143 | |
144 | cq = get_qedr_cq(ibcq: attrs->recv_cq); |
145 | /* if a dedicated recv_cq was used, delete it too */ |
146 | if (iparams.icid != cq->icid) { |
147 | iparams.icid = cq->icid; |
148 | dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); |
149 | dev->ops->common->chain_free(dev->cdev, &cq->pbl); |
150 | } |
151 | } |
152 | |
153 | static inline int qedr_check_gsi_qp_attrs(struct qedr_dev *dev, |
154 | struct ib_qp_init_attr *attrs) |
155 | { |
156 | if (attrs->cap.max_recv_sge > QEDR_GSI_MAX_RECV_SGE) { |
157 | DP_ERR(dev, |
158 | " create gsi qp: failed. max_recv_sge is larger the max %d>%d\n" , |
159 | attrs->cap.max_recv_sge, QEDR_GSI_MAX_RECV_SGE); |
160 | return -EINVAL; |
161 | } |
162 | |
163 | if (attrs->cap.max_recv_wr > QEDR_GSI_MAX_RECV_WR) { |
164 | DP_ERR(dev, |
165 | " create gsi qp: failed. max_recv_wr is too large %d>%d\n" , |
166 | attrs->cap.max_recv_wr, QEDR_GSI_MAX_RECV_WR); |
167 | return -EINVAL; |
168 | } |
169 | |
170 | if (attrs->cap.max_send_wr > QEDR_GSI_MAX_SEND_WR) { |
171 | DP_ERR(dev, |
172 | " create gsi qp: failed. max_send_wr is too large %d>%d\n" , |
173 | attrs->cap.max_send_wr, QEDR_GSI_MAX_SEND_WR); |
174 | return -EINVAL; |
175 | } |
176 | |
177 | return 0; |
178 | } |
179 | |
180 | static int qedr_ll2_post_tx(struct qedr_dev *dev, |
181 | struct qed_roce_ll2_packet *pkt) |
182 | { |
183 | enum qed_ll2_roce_flavor_type roce_flavor; |
184 | struct qed_ll2_tx_pkt_info ll2_tx_pkt; |
185 | int rc; |
186 | int i; |
187 | |
188 | memset(&ll2_tx_pkt, 0, sizeof(ll2_tx_pkt)); |
189 | |
190 | roce_flavor = (pkt->roce_mode == ROCE_V1) ? |
191 | QED_LL2_ROCE : QED_LL2_RROCE; |
192 | |
193 | if (pkt->roce_mode == ROCE_V2_IPV4) |
194 | ll2_tx_pkt.enable_ip_cksum = 1; |
195 | |
196 | ll2_tx_pkt.num_of_bds = 1 /* hdr */ + pkt->n_seg; |
197 | ll2_tx_pkt.vlan = 0; |
198 | ll2_tx_pkt.tx_dest = pkt->tx_dest; |
199 | ll2_tx_pkt.qed_roce_flavor = roce_flavor; |
200 | ll2_tx_pkt.first_frag = pkt->header.baddr; |
201 | ll2_tx_pkt.first_frag_len = pkt->header.len; |
202 | ll2_tx_pkt.cookie = pkt; |
203 | |
204 | /* tx header */ |
205 | rc = dev->ops->ll2_prepare_tx_packet(dev->rdma_ctx, |
206 | dev->gsi_ll2_handle, |
207 | &ll2_tx_pkt, 1); |
208 | if (rc) { |
209 | /* TX failed while posting header - release resources */ |
210 | dma_free_coherent(dev: &dev->pdev->dev, size: pkt->header.len, |
211 | cpu_addr: pkt->header.vaddr, dma_handle: pkt->header.baddr); |
212 | kfree(objp: pkt); |
213 | |
214 | DP_ERR(dev, "roce ll2 tx: header failed (rc=%d)\n" , rc); |
215 | return rc; |
216 | } |
217 | |
218 | /* tx payload */ |
219 | for (i = 0; i < pkt->n_seg; i++) { |
220 | rc = dev->ops->ll2_set_fragment_of_tx_packet( |
221 | dev->rdma_ctx, |
222 | dev->gsi_ll2_handle, |
223 | pkt->payload[i].baddr, |
224 | pkt->payload[i].len); |
225 | |
226 | if (rc) { |
227 | /* if failed not much to do here, partial packet has |
228 | * been posted we can't free memory, will need to wait |
229 | * for completion |
230 | */ |
231 | DP_ERR(dev, "ll2 tx: payload failed (rc=%d)\n" , rc); |
232 | return rc; |
233 | } |
234 | } |
235 | |
236 | return 0; |
237 | } |
238 | |
239 | static int qedr_ll2_stop(struct qedr_dev *dev) |
240 | { |
241 | int rc; |
242 | |
243 | if (dev->gsi_ll2_handle == QED_LL2_UNUSED_HANDLE) |
244 | return 0; |
245 | |
246 | /* remove LL2 MAC address filter */ |
247 | rc = dev->ops->ll2_set_mac_filter(dev->cdev, |
248 | dev->gsi_ll2_mac_address, NULL); |
249 | |
250 | rc = dev->ops->ll2_terminate_connection(dev->rdma_ctx, |
251 | dev->gsi_ll2_handle); |
252 | if (rc) |
253 | DP_ERR(dev, "Failed to terminate LL2 connection (rc=%d)\n" , rc); |
254 | |
255 | dev->ops->ll2_release_connection(dev->rdma_ctx, dev->gsi_ll2_handle); |
256 | |
257 | dev->gsi_ll2_handle = QED_LL2_UNUSED_HANDLE; |
258 | |
259 | return rc; |
260 | } |
261 | |
262 | static int qedr_ll2_start(struct qedr_dev *dev, |
263 | struct ib_qp_init_attr *attrs, struct qedr_qp *qp) |
264 | { |
265 | struct qed_ll2_acquire_data data; |
266 | struct qed_ll2_cbs cbs; |
267 | int rc; |
268 | |
269 | /* configure and start LL2 */ |
270 | cbs.rx_comp_cb = qedr_ll2_complete_rx_packet; |
271 | cbs.tx_comp_cb = qedr_ll2_complete_tx_packet; |
272 | cbs.rx_release_cb = qedr_ll2_release_rx_packet; |
273 | cbs.tx_release_cb = qedr_ll2_complete_tx_packet; |
274 | cbs.cookie = dev; |
275 | |
276 | memset(&data, 0, sizeof(data)); |
277 | data.input.conn_type = QED_LL2_TYPE_ROCE; |
278 | data.input.mtu = dev->ndev->mtu; |
279 | data.input.rx_num_desc = attrs->cap.max_recv_wr; |
280 | data.input.rx_drop_ttl0_flg = true; |
281 | data.input.rx_vlan_removal_en = false; |
282 | data.input.tx_num_desc = attrs->cap.max_send_wr; |
283 | data.input.tx_tc = 0; |
284 | data.input.tx_dest = QED_LL2_TX_DEST_NW; |
285 | data.input.ai_err_packet_too_big = QED_LL2_DROP_PACKET; |
286 | data.input.ai_err_no_buf = QED_LL2_DROP_PACKET; |
287 | data.input.gsi_enable = 1; |
288 | data.p_connection_handle = &dev->gsi_ll2_handle; |
289 | data.cbs = &cbs; |
290 | |
291 | rc = dev->ops->ll2_acquire_connection(dev->rdma_ctx, &data); |
292 | if (rc) { |
293 | DP_ERR(dev, |
294 | "ll2 start: failed to acquire LL2 connection (rc=%d)\n" , |
295 | rc); |
296 | return rc; |
297 | } |
298 | |
299 | rc = dev->ops->ll2_establish_connection(dev->rdma_ctx, |
300 | dev->gsi_ll2_handle); |
301 | if (rc) { |
302 | DP_ERR(dev, |
303 | "ll2 start: failed to establish LL2 connection (rc=%d)\n" , |
304 | rc); |
305 | goto err1; |
306 | } |
307 | |
308 | rc = dev->ops->ll2_set_mac_filter(dev->cdev, NULL, dev->ndev->dev_addr); |
309 | if (rc) |
310 | goto err2; |
311 | |
312 | return 0; |
313 | |
314 | err2: |
315 | dev->ops->ll2_terminate_connection(dev->rdma_ctx, dev->gsi_ll2_handle); |
316 | err1: |
317 | dev->ops->ll2_release_connection(dev->rdma_ctx, dev->gsi_ll2_handle); |
318 | |
319 | return rc; |
320 | } |
321 | |
322 | int qedr_create_gsi_qp(struct qedr_dev *dev, struct ib_qp_init_attr *attrs, |
323 | struct qedr_qp *qp) |
324 | { |
325 | int rc; |
326 | |
327 | rc = qedr_check_gsi_qp_attrs(dev, attrs); |
328 | if (rc) |
329 | return rc; |
330 | |
331 | rc = qedr_ll2_start(dev, attrs, qp); |
332 | if (rc) { |
333 | DP_ERR(dev, "create gsi qp: failed on ll2 start. rc=%d\n" , rc); |
334 | return rc; |
335 | } |
336 | |
337 | /* create QP */ |
338 | qp->ibqp.qp_num = 1; |
339 | qp->rq.max_wr = attrs->cap.max_recv_wr; |
340 | qp->sq.max_wr = attrs->cap.max_send_wr; |
341 | |
342 | qp->rqe_wr_id = kcalloc(n: qp->rq.max_wr, size: sizeof(*qp->rqe_wr_id), |
343 | GFP_KERNEL); |
344 | if (!qp->rqe_wr_id) |
345 | goto err; |
346 | qp->wqe_wr_id = kcalloc(n: qp->sq.max_wr, size: sizeof(*qp->wqe_wr_id), |
347 | GFP_KERNEL); |
348 | if (!qp->wqe_wr_id) |
349 | goto err; |
350 | |
351 | qedr_store_gsi_qp_cq(dev, qp, attrs); |
352 | ether_addr_copy(dst: dev->gsi_ll2_mac_address, src: dev->ndev->dev_addr); |
353 | |
354 | /* the GSI CQ is handled by the driver so remove it from the FW */ |
355 | qedr_destroy_gsi_cq(dev, attrs); |
356 | dev->gsi_rqcq->cq_type = QEDR_CQ_TYPE_GSI; |
357 | |
358 | DP_DEBUG(dev, QEDR_MSG_GSI, "created GSI QP %p\n" , qp); |
359 | |
360 | return 0; |
361 | |
362 | err: |
363 | kfree(objp: qp->rqe_wr_id); |
364 | |
365 | rc = qedr_ll2_stop(dev); |
366 | if (rc) |
367 | DP_ERR(dev, "create gsi qp: failed destroy on create\n" ); |
368 | |
369 | return -ENOMEM; |
370 | } |
371 | |
372 | int qedr_destroy_gsi_qp(struct qedr_dev *dev) |
373 | { |
374 | return qedr_ll2_stop(dev); |
375 | } |
376 | |
377 | #define (100) |
378 | #define QEDR_GSI_QPN (1) |
379 | static inline int (struct qedr_dev *dev, |
380 | struct qedr_qp *qp, |
381 | const struct ib_send_wr *swr, |
382 | struct ib_ud_header *udh, |
383 | int *roce_mode) |
384 | { |
385 | bool has_vlan = false, has_grh_ipv6 = true; |
386 | struct rdma_ah_attr *ah_attr = &get_qedr_ah(ibah: ud_wr(wr: swr)->ah)->attr; |
387 | const struct ib_global_route *grh = rdma_ah_read_grh(attr: ah_attr); |
388 | const struct ib_gid_attr *sgid_attr = grh->sgid_attr; |
389 | int send_size = 0; |
390 | u16 vlan_id = 0; |
391 | u16 ether_type; |
392 | int rc; |
393 | int ip_ver = 0; |
394 | |
395 | bool has_udp = false; |
396 | int i; |
397 | |
398 | rc = rdma_read_gid_l2_fields(attr: sgid_attr, vlan_id: &vlan_id, NULL); |
399 | if (rc) |
400 | return rc; |
401 | |
402 | if (vlan_id < VLAN_CFI_MASK) |
403 | has_vlan = true; |
404 | |
405 | send_size = 0; |
406 | for (i = 0; i < swr->num_sge; ++i) |
407 | send_size += swr->sg_list[i].length; |
408 | |
409 | has_udp = (sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP); |
410 | if (!has_udp) { |
411 | /* RoCE v1 */ |
412 | ether_type = ETH_P_IBOE; |
413 | *roce_mode = ROCE_V1; |
414 | } else if (ipv6_addr_v4mapped(a: (struct in6_addr *)&sgid_attr->gid)) { |
415 | /* RoCE v2 IPv4 */ |
416 | ip_ver = 4; |
417 | ether_type = ETH_P_IP; |
418 | has_grh_ipv6 = false; |
419 | *roce_mode = ROCE_V2_IPV4; |
420 | } else { |
421 | /* RoCE v2 IPv6 */ |
422 | ip_ver = 6; |
423 | ether_type = ETH_P_IPV6; |
424 | *roce_mode = ROCE_V2_IPV6; |
425 | } |
426 | |
427 | rc = ib_ud_header_init(payload_bytes: send_size, lrh_present: false, eth_present: true, vlan_present: has_vlan, |
428 | grh_present: has_grh_ipv6, ip_version: ip_ver, udp_present: has_udp, immediate_present: 0, header: udh); |
429 | if (rc) { |
430 | DP_ERR(dev, "gsi post send: failed to init header\n" ); |
431 | return rc; |
432 | } |
433 | |
434 | /* ENET + VLAN headers */ |
435 | ether_addr_copy(dst: udh->eth.dmac_h, src: ah_attr->roce.dmac); |
436 | ether_addr_copy(dst: udh->eth.smac_h, src: dev->ndev->dev_addr); |
437 | if (has_vlan) { |
438 | udh->eth.type = htons(ETH_P_8021Q); |
439 | udh->vlan.tag = htons(vlan_id); |
440 | udh->vlan.type = htons(ether_type); |
441 | } else { |
442 | udh->eth.type = htons(ether_type); |
443 | } |
444 | |
445 | /* BTH */ |
446 | udh->bth.solicited_event = !!(swr->send_flags & IB_SEND_SOLICITED); |
447 | udh->bth.pkey = QEDR_ROCE_PKEY_DEFAULT; |
448 | udh->bth.destination_qpn = htonl(ud_wr(swr)->remote_qpn); |
449 | udh->bth.psn = htonl((qp->sq_psn++) & ((1 << 24) - 1)); |
450 | udh->bth.opcode = IB_OPCODE_UD_SEND_ONLY; |
451 | |
452 | /* DETH */ |
453 | udh->deth.qkey = htonl(0x80010000); |
454 | udh->deth.source_qpn = htonl(QEDR_GSI_QPN); |
455 | |
456 | if (has_grh_ipv6) { |
457 | /* GRH / IPv6 header */ |
458 | udh->grh.traffic_class = grh->traffic_class; |
459 | udh->grh.flow_label = grh->flow_label; |
460 | udh->grh.hop_limit = grh->hop_limit; |
461 | udh->grh.destination_gid = grh->dgid; |
462 | memcpy(&udh->grh.source_gid.raw, sgid_attr->gid.raw, |
463 | sizeof(udh->grh.source_gid.raw)); |
464 | } else { |
465 | /* IPv4 header */ |
466 | u32 ipv4_addr; |
467 | |
468 | udh->ip4.protocol = IPPROTO_UDP; |
469 | udh->ip4.tos = htonl(grh->flow_label); |
470 | udh->ip4.frag_off = htons(IP_DF); |
471 | udh->ip4.ttl = grh->hop_limit; |
472 | |
473 | ipv4_addr = qedr_get_ipv4_from_gid(gid: sgid_attr->gid.raw); |
474 | udh->ip4.saddr = ipv4_addr; |
475 | ipv4_addr = qedr_get_ipv4_from_gid(gid: grh->dgid.raw); |
476 | udh->ip4.daddr = ipv4_addr; |
477 | /* note: checksum is calculated by the device */ |
478 | } |
479 | |
480 | /* UDP */ |
481 | if (has_udp) { |
482 | udh->udp.sport = htons(QEDR_ROCE_V2_UDP_SPORT); |
483 | udh->udp.dport = htons(ROCE_V2_UDP_DPORT); |
484 | udh->udp.csum = 0; |
485 | /* UDP length is untouched hence is zero */ |
486 | } |
487 | return 0; |
488 | } |
489 | |
490 | static inline int qedr_gsi_build_packet(struct qedr_dev *dev, |
491 | struct qedr_qp *qp, |
492 | const struct ib_send_wr *swr, |
493 | struct qed_roce_ll2_packet **p_packet) |
494 | { |
495 | u8 [QEDR_MAX_UD_HEADER_SIZE]; |
496 | struct qed_roce_ll2_packet *packet; |
497 | struct pci_dev *pdev = dev->pdev; |
498 | int roce_mode, ; |
499 | struct ib_ud_header udh; |
500 | int i, rc; |
501 | |
502 | *p_packet = NULL; |
503 | |
504 | rc = qedr_gsi_build_header(dev, qp, swr, udh: &udh, roce_mode: &roce_mode); |
505 | if (rc) |
506 | return rc; |
507 | |
508 | header_size = ib_ud_header_pack(header: &udh, buf: &ud_header_buffer); |
509 | |
510 | packet = kzalloc(size: sizeof(*packet), GFP_ATOMIC); |
511 | if (!packet) |
512 | return -ENOMEM; |
513 | |
514 | packet->header.vaddr = dma_alloc_coherent(dev: &pdev->dev, size: header_size, |
515 | dma_handle: &packet->header.baddr, |
516 | GFP_ATOMIC); |
517 | if (!packet->header.vaddr) { |
518 | kfree(objp: packet); |
519 | return -ENOMEM; |
520 | } |
521 | |
522 | if (ether_addr_equal(addr1: udh.eth.smac_h, addr2: udh.eth.dmac_h)) |
523 | packet->tx_dest = QED_LL2_TX_DEST_LB; |
524 | else |
525 | packet->tx_dest = QED_LL2_TX_DEST_NW; |
526 | |
527 | packet->roce_mode = roce_mode; |
528 | memcpy(packet->header.vaddr, ud_header_buffer, header_size); |
529 | packet->header.len = header_size; |
530 | packet->n_seg = swr->num_sge; |
531 | for (i = 0; i < packet->n_seg; i++) { |
532 | packet->payload[i].baddr = swr->sg_list[i].addr; |
533 | packet->payload[i].len = swr->sg_list[i].length; |
534 | } |
535 | |
536 | *p_packet = packet; |
537 | |
538 | return 0; |
539 | } |
540 | |
541 | int qedr_gsi_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, |
542 | const struct ib_send_wr **bad_wr) |
543 | { |
544 | struct qed_roce_ll2_packet *pkt = NULL; |
545 | struct qedr_qp *qp = get_qedr_qp(ibqp); |
546 | struct qedr_dev *dev = qp->dev; |
547 | unsigned long flags; |
548 | int rc; |
549 | |
550 | if (qp->state != QED_ROCE_QP_STATE_RTS) { |
551 | *bad_wr = wr; |
552 | DP_ERR(dev, |
553 | "gsi post recv: failed to post rx buffer. state is %d and not QED_ROCE_QP_STATE_RTS\n" , |
554 | qp->state); |
555 | return -EINVAL; |
556 | } |
557 | |
558 | if (wr->num_sge > RDMA_MAX_SGE_PER_SQ_WQE) { |
559 | DP_ERR(dev, "gsi post send: num_sge is too large (%d>%d)\n" , |
560 | wr->num_sge, RDMA_MAX_SGE_PER_SQ_WQE); |
561 | rc = -EINVAL; |
562 | goto err; |
563 | } |
564 | |
565 | if (wr->opcode != IB_WR_SEND) { |
566 | DP_ERR(dev, |
567 | "gsi post send: failed due to unsupported opcode %d\n" , |
568 | wr->opcode); |
569 | rc = -EINVAL; |
570 | goto err; |
571 | } |
572 | |
573 | spin_lock_irqsave(&qp->q_lock, flags); |
574 | |
575 | rc = qedr_gsi_build_packet(dev, qp, swr: wr, p_packet: &pkt); |
576 | if (rc) { |
577 | spin_unlock_irqrestore(lock: &qp->q_lock, flags); |
578 | goto err; |
579 | } |
580 | |
581 | rc = qedr_ll2_post_tx(dev, pkt); |
582 | |
583 | if (!rc) { |
584 | qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; |
585 | qedr_inc_sw_prod(info: &qp->sq); |
586 | DP_DEBUG(qp->dev, QEDR_MSG_GSI, |
587 | "gsi post send: opcode=%d, wr_id=%llx\n" , wr->opcode, |
588 | wr->wr_id); |
589 | } else { |
590 | DP_ERR(dev, "gsi post send: failed to transmit (rc=%d)\n" , rc); |
591 | rc = -EAGAIN; |
592 | *bad_wr = wr; |
593 | } |
594 | |
595 | spin_unlock_irqrestore(lock: &qp->q_lock, flags); |
596 | |
597 | if (wr->next) { |
598 | DP_ERR(dev, |
599 | "gsi post send: failed second WR. Only one WR may be passed at a time\n" ); |
600 | *bad_wr = wr->next; |
601 | rc = -EINVAL; |
602 | } |
603 | |
604 | return rc; |
605 | |
606 | err: |
607 | *bad_wr = wr; |
608 | return rc; |
609 | } |
610 | |
611 | int qedr_gsi_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, |
612 | const struct ib_recv_wr **bad_wr) |
613 | { |
614 | struct qedr_dev *dev = get_qedr_dev(ibdev: ibqp->device); |
615 | struct qedr_qp *qp = get_qedr_qp(ibqp); |
616 | unsigned long flags; |
617 | int rc = 0; |
618 | |
619 | if ((qp->state != QED_ROCE_QP_STATE_RTR) && |
620 | (qp->state != QED_ROCE_QP_STATE_RTS)) { |
621 | *bad_wr = wr; |
622 | DP_ERR(dev, |
623 | "gsi post recv: failed to post rx buffer. state is %d and not QED_ROCE_QP_STATE_RTR/S\n" , |
624 | qp->state); |
625 | return -EINVAL; |
626 | } |
627 | |
628 | spin_lock_irqsave(&qp->q_lock, flags); |
629 | |
630 | while (wr) { |
631 | if (wr->num_sge > QEDR_GSI_MAX_RECV_SGE) { |
632 | DP_ERR(dev, |
633 | "gsi post recv: failed to post rx buffer. too many sges %d>%d\n" , |
634 | wr->num_sge, QEDR_GSI_MAX_RECV_SGE); |
635 | goto err; |
636 | } |
637 | |
638 | rc = dev->ops->ll2_post_rx_buffer(dev->rdma_ctx, |
639 | dev->gsi_ll2_handle, |
640 | wr->sg_list[0].addr, |
641 | wr->sg_list[0].length, |
642 | NULL /* cookie */, |
643 | 1 /* notify_fw */); |
644 | if (rc) { |
645 | DP_ERR(dev, |
646 | "gsi post recv: failed to post rx buffer (rc=%d)\n" , |
647 | rc); |
648 | goto err; |
649 | } |
650 | |
651 | memset(&qp->rqe_wr_id[qp->rq.prod], 0, |
652 | sizeof(qp->rqe_wr_id[qp->rq.prod])); |
653 | qp->rqe_wr_id[qp->rq.prod].sg_list[0] = wr->sg_list[0]; |
654 | qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; |
655 | |
656 | qedr_inc_sw_prod(info: &qp->rq); |
657 | |
658 | wr = wr->next; |
659 | } |
660 | |
661 | spin_unlock_irqrestore(lock: &qp->q_lock, flags); |
662 | |
663 | return rc; |
664 | err: |
665 | spin_unlock_irqrestore(lock: &qp->q_lock, flags); |
666 | *bad_wr = wr; |
667 | return -ENOMEM; |
668 | } |
669 | |
670 | int qedr_gsi_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) |
671 | { |
672 | struct qedr_dev *dev = get_qedr_dev(ibdev: ibcq->device); |
673 | struct qedr_cq *cq = get_qedr_cq(ibcq); |
674 | struct qedr_qp *qp = dev->gsi_qp; |
675 | unsigned long flags; |
676 | u16 vlan_id; |
677 | int i = 0; |
678 | |
679 | spin_lock_irqsave(&cq->cq_lock, flags); |
680 | |
681 | while (i < num_entries && qp->rq.cons != qp->rq.gsi_cons) { |
682 | memset(&wc[i], 0, sizeof(*wc)); |
683 | |
684 | wc[i].qp = &qp->ibqp; |
685 | wc[i].wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; |
686 | wc[i].opcode = IB_WC_RECV; |
687 | wc[i].pkey_index = 0; |
688 | wc[i].status = (qp->rqe_wr_id[qp->rq.cons].rc) ? |
689 | IB_WC_GENERAL_ERR : IB_WC_SUCCESS; |
690 | /* 0 - currently only one recv sg is supported */ |
691 | wc[i].byte_len = qp->rqe_wr_id[qp->rq.cons].sg_list[0].length; |
692 | wc[i].wc_flags |= IB_WC_GRH | IB_WC_IP_CSUM_OK; |
693 | ether_addr_copy(dst: wc[i].smac, src: qp->rqe_wr_id[qp->rq.cons].smac); |
694 | wc[i].wc_flags |= IB_WC_WITH_SMAC; |
695 | |
696 | vlan_id = qp->rqe_wr_id[qp->rq.cons].vlan & |
697 | VLAN_VID_MASK; |
698 | if (vlan_id) { |
699 | wc[i].wc_flags |= IB_WC_WITH_VLAN; |
700 | wc[i].vlan_id = vlan_id; |
701 | wc[i].sl = (qp->rqe_wr_id[qp->rq.cons].vlan & |
702 | VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; |
703 | } |
704 | |
705 | qedr_inc_sw_cons(info: &qp->rq); |
706 | i++; |
707 | } |
708 | |
709 | while (i < num_entries && qp->sq.cons != qp->sq.gsi_cons) { |
710 | memset(&wc[i], 0, sizeof(*wc)); |
711 | |
712 | wc[i].qp = &qp->ibqp; |
713 | wc[i].wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; |
714 | wc[i].opcode = IB_WC_SEND; |
715 | wc[i].status = IB_WC_SUCCESS; |
716 | |
717 | qedr_inc_sw_cons(info: &qp->sq); |
718 | i++; |
719 | } |
720 | |
721 | spin_unlock_irqrestore(lock: &cq->cq_lock, flags); |
722 | |
723 | DP_DEBUG(dev, QEDR_MSG_GSI, |
724 | "gsi poll_cq: requested entries=%d, actual=%d, qp->rq.cons=%d, qp->rq.gsi_cons=%x, qp->sq.cons=%d, qp->sq.gsi_cons=%d, qp_num=%d\n" , |
725 | num_entries, i, qp->rq.cons, qp->rq.gsi_cons, qp->sq.cons, |
726 | qp->sq.gsi_cons, qp->ibqp.qp_num); |
727 | |
728 | return i; |
729 | } |
730 | |