1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (c) 2016 Chelsio Communications, Inc. |
4 | */ |
5 | |
6 | #define DRV_NAME "cxgbit" |
7 | #define DRV_VERSION "1.0.0-ko" |
8 | #define pr_fmt(fmt) DRV_NAME ": " fmt |
9 | |
10 | #include "cxgbit.h" |
11 | |
12 | #ifdef CONFIG_CHELSIO_T4_DCB |
13 | #include <net/dcbevent.h> |
14 | #include "cxgb4_dcb.h" |
15 | #endif |
16 | |
17 | LIST_HEAD(cdev_list_head); |
18 | /* cdev list lock */ |
19 | DEFINE_MUTEX(cdev_list_lock); |
20 | |
21 | void _cxgbit_free_cdev(struct kref *kref) |
22 | { |
23 | struct cxgbit_device *cdev; |
24 | |
25 | cdev = container_of(kref, struct cxgbit_device, kref); |
26 | |
27 | cxgbi_ppm_release(cdev2ppm(cdev)); |
28 | kfree(objp: cdev); |
29 | } |
30 | |
31 | static void cxgbit_set_mdsl(struct cxgbit_device *cdev) |
32 | { |
33 | struct cxgb4_lld_info *lldi = &cdev->lldi; |
34 | u32 mdsl; |
35 | |
36 | #define CXGBIT_T5_MAX_PDU_LEN 16224 |
37 | #define CXGBIT_PDU_NONPAYLOAD_LEN 312 /* 48(BHS) + 256(AHS) + 8(Digest) */ |
38 | if (is_t5(lldi->adapter_type)) { |
39 | mdsl = min_t(u32, lldi->iscsi_iolen - CXGBIT_PDU_NONPAYLOAD_LEN, |
40 | CXGBIT_T5_MAX_PDU_LEN - CXGBIT_PDU_NONPAYLOAD_LEN); |
41 | } else { |
42 | mdsl = lldi->iscsi_iolen - CXGBIT_PDU_NONPAYLOAD_LEN; |
43 | mdsl = min(mdsl, 16384U); |
44 | } |
45 | |
46 | mdsl = round_down(mdsl, 4); |
47 | mdsl = min_t(u32, mdsl, 4 * PAGE_SIZE); |
48 | mdsl = min_t(u32, mdsl, (MAX_SKB_FRAGS - 1) * PAGE_SIZE); |
49 | |
50 | cdev->mdsl = mdsl; |
51 | } |
52 | |
53 | static void *cxgbit_uld_add(const struct cxgb4_lld_info *lldi) |
54 | { |
55 | struct cxgbit_device *cdev; |
56 | |
57 | if (is_t4(lldi->adapter_type)) |
58 | return ERR_PTR(error: -ENODEV); |
59 | |
60 | cdev = kzalloc(size: sizeof(*cdev), GFP_KERNEL); |
61 | if (!cdev) |
62 | return ERR_PTR(error: -ENOMEM); |
63 | |
64 | kref_init(kref: &cdev->kref); |
65 | spin_lock_init(&cdev->np_lock); |
66 | |
67 | cdev->lldi = *lldi; |
68 | |
69 | cxgbit_set_mdsl(cdev); |
70 | |
71 | if (cxgbit_ddp_init(cdev) < 0) { |
72 | kfree(objp: cdev); |
73 | return ERR_PTR(error: -EINVAL); |
74 | } |
75 | |
76 | if (!test_bit(CDEV_DDP_ENABLE, &cdev->flags)) |
77 | pr_info("cdev %s ddp init failed\n" , |
78 | pci_name(lldi->pdev)); |
79 | |
80 | if (lldi->fw_vers >= 0x10d2b00) |
81 | set_bit(nr: CDEV_ISO_ENABLE, addr: &cdev->flags); |
82 | |
83 | spin_lock_init(&cdev->cskq.lock); |
84 | INIT_LIST_HEAD(list: &cdev->cskq.list); |
85 | |
86 | mutex_lock(&cdev_list_lock); |
87 | list_add_tail(new: &cdev->list, head: &cdev_list_head); |
88 | mutex_unlock(lock: &cdev_list_lock); |
89 | |
90 | pr_info("cdev %s added for iSCSI target transport\n" , |
91 | pci_name(lldi->pdev)); |
92 | |
93 | return cdev; |
94 | } |
95 | |
96 | static void cxgbit_close_conn(struct cxgbit_device *cdev) |
97 | { |
98 | struct cxgbit_sock *csk; |
99 | struct sk_buff *skb; |
100 | bool wakeup_thread = false; |
101 | |
102 | spin_lock_bh(lock: &cdev->cskq.lock); |
103 | list_for_each_entry(csk, &cdev->cskq.list, list) { |
104 | skb = alloc_skb(size: 0, GFP_ATOMIC); |
105 | if (!skb) |
106 | continue; |
107 | |
108 | spin_lock_bh(lock: &csk->rxq.lock); |
109 | __skb_queue_tail(list: &csk->rxq, newsk: skb); |
110 | if (skb_queue_len(list_: &csk->rxq) == 1) |
111 | wakeup_thread = true; |
112 | spin_unlock_bh(lock: &csk->rxq.lock); |
113 | |
114 | if (wakeup_thread) { |
115 | wake_up(&csk->waitq); |
116 | wakeup_thread = false; |
117 | } |
118 | } |
119 | spin_unlock_bh(lock: &cdev->cskq.lock); |
120 | } |
121 | |
122 | static void cxgbit_detach_cdev(struct cxgbit_device *cdev) |
123 | { |
124 | bool free_cdev = false; |
125 | |
126 | spin_lock_bh(lock: &cdev->cskq.lock); |
127 | if (list_empty(head: &cdev->cskq.list)) |
128 | free_cdev = true; |
129 | spin_unlock_bh(lock: &cdev->cskq.lock); |
130 | |
131 | if (free_cdev) { |
132 | mutex_lock(&cdev_list_lock); |
133 | list_del(entry: &cdev->list); |
134 | mutex_unlock(lock: &cdev_list_lock); |
135 | |
136 | cxgbit_put_cdev(cdev); |
137 | } else { |
138 | cxgbit_close_conn(cdev); |
139 | } |
140 | } |
141 | |
142 | static int cxgbit_uld_state_change(void *handle, enum cxgb4_state state) |
143 | { |
144 | struct cxgbit_device *cdev = handle; |
145 | |
146 | switch (state) { |
147 | case CXGB4_STATE_UP: |
148 | set_bit(nr: CDEV_STATE_UP, addr: &cdev->flags); |
149 | pr_info("cdev %s state UP.\n" , pci_name(cdev->lldi.pdev)); |
150 | break; |
151 | case CXGB4_STATE_START_RECOVERY: |
152 | clear_bit(nr: CDEV_STATE_UP, addr: &cdev->flags); |
153 | cxgbit_close_conn(cdev); |
154 | pr_info("cdev %s state RECOVERY.\n" , pci_name(cdev->lldi.pdev)); |
155 | break; |
156 | case CXGB4_STATE_DOWN: |
157 | pr_info("cdev %s state DOWN.\n" , pci_name(cdev->lldi.pdev)); |
158 | break; |
159 | case CXGB4_STATE_DETACH: |
160 | clear_bit(nr: CDEV_STATE_UP, addr: &cdev->flags); |
161 | pr_info("cdev %s state DETACH.\n" , pci_name(cdev->lldi.pdev)); |
162 | cxgbit_detach_cdev(cdev); |
163 | break; |
164 | default: |
165 | pr_info("cdev %s unknown state %d.\n" , |
166 | pci_name(cdev->lldi.pdev), state); |
167 | break; |
168 | } |
169 | return 0; |
170 | } |
171 | |
172 | static void |
173 | cxgbit_process_ddpvld(struct cxgbit_sock *csk, struct cxgbit_lro_pdu_cb *pdu_cb, |
174 | u32 ddpvld) |
175 | { |
176 | |
177 | if (ddpvld & (1 << CPL_RX_ISCSI_DDP_STATUS_HCRC_SHIFT)) { |
178 | pr_info("tid 0x%x, status 0x%x, hcrc bad.\n" , csk->tid, ddpvld); |
179 | pdu_cb->flags |= PDUCBF_RX_HCRC_ERR; |
180 | } |
181 | |
182 | if (ddpvld & (1 << CPL_RX_ISCSI_DDP_STATUS_DCRC_SHIFT)) { |
183 | pr_info("tid 0x%x, status 0x%x, dcrc bad.\n" , csk->tid, ddpvld); |
184 | pdu_cb->flags |= PDUCBF_RX_DCRC_ERR; |
185 | } |
186 | |
187 | if (ddpvld & (1 << CPL_RX_ISCSI_DDP_STATUS_PAD_SHIFT)) |
188 | pr_info("tid 0x%x, status 0x%x, pad bad.\n" , csk->tid, ddpvld); |
189 | |
190 | if ((ddpvld & (1 << CPL_RX_ISCSI_DDP_STATUS_DDP_SHIFT)) && |
191 | (!(pdu_cb->flags & PDUCBF_RX_DATA))) { |
192 | pdu_cb->flags |= PDUCBF_RX_DATA_DDPD; |
193 | } |
194 | } |
195 | |
196 | static void |
197 | cxgbit_lro_add_packet_rsp(struct sk_buff *skb, u8 op, const __be64 *rsp) |
198 | { |
199 | struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); |
200 | struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, |
201 | lro_cb->pdu_idx); |
202 | struct cpl_rx_iscsi_ddp *cpl = (struct cpl_rx_iscsi_ddp *)(rsp + 1); |
203 | |
204 | cxgbit_process_ddpvld(csk: lro_cb->csk, pdu_cb, be32_to_cpu(cpl->ddpvld)); |
205 | |
206 | pdu_cb->flags |= PDUCBF_RX_STATUS; |
207 | pdu_cb->ddigest = ntohl(cpl->ulp_crc); |
208 | pdu_cb->pdulen = ntohs(cpl->len); |
209 | |
210 | if (pdu_cb->flags & PDUCBF_RX_HDR) |
211 | pdu_cb->complete = true; |
212 | |
213 | lro_cb->pdu_totallen += pdu_cb->pdulen; |
214 | lro_cb->complete = true; |
215 | lro_cb->pdu_idx++; |
216 | } |
217 | |
218 | static void |
219 | cxgbit_copy_frags(struct sk_buff *skb, const struct pkt_gl *gl, |
220 | unsigned int offset) |
221 | { |
222 | u8 skb_frag_idx = skb_shinfo(skb)->nr_frags; |
223 | u8 i; |
224 | |
225 | /* usually there's just one frag */ |
226 | __skb_fill_page_desc(skb, i: skb_frag_idx, page: gl->frags[0].page, |
227 | off: gl->frags[0].offset + offset, |
228 | size: gl->frags[0].size - offset); |
229 | for (i = 1; i < gl->nfrags; i++) |
230 | __skb_fill_page_desc(skb, i: skb_frag_idx + i, |
231 | page: gl->frags[i].page, |
232 | off: gl->frags[i].offset, |
233 | size: gl->frags[i].size); |
234 | |
235 | skb_shinfo(skb)->nr_frags += gl->nfrags; |
236 | |
237 | /* get a reference to the last page, we don't own it */ |
238 | get_page(page: gl->frags[gl->nfrags - 1].page); |
239 | } |
240 | |
241 | static void |
242 | cxgbit_lro_add_packet_gl(struct sk_buff *skb, u8 op, const struct pkt_gl *gl) |
243 | { |
244 | struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); |
245 | struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, |
246 | lro_cb->pdu_idx); |
247 | u32 len, offset; |
248 | |
249 | if (op == CPL_ISCSI_HDR) { |
250 | struct cpl_iscsi_hdr *cpl = (struct cpl_iscsi_hdr *)gl->va; |
251 | |
252 | offset = sizeof(struct cpl_iscsi_hdr); |
253 | pdu_cb->flags |= PDUCBF_RX_HDR; |
254 | pdu_cb->seq = ntohl(cpl->seq); |
255 | len = ntohs(cpl->len); |
256 | pdu_cb->hdr = gl->va + offset; |
257 | pdu_cb->hlen = len; |
258 | pdu_cb->hfrag_idx = skb_shinfo(skb)->nr_frags; |
259 | |
260 | if (unlikely(gl->nfrags > 1)) |
261 | cxgbit_skcb_flags(skb) = 0; |
262 | |
263 | lro_cb->complete = false; |
264 | } else if (op == CPL_ISCSI_DATA) { |
265 | struct cpl_iscsi_data *cpl = (struct cpl_iscsi_data *)gl->va; |
266 | |
267 | offset = sizeof(struct cpl_iscsi_data); |
268 | pdu_cb->flags |= PDUCBF_RX_DATA; |
269 | len = ntohs(cpl->len); |
270 | pdu_cb->dlen = len; |
271 | pdu_cb->doffset = lro_cb->offset; |
272 | pdu_cb->nr_dfrags = gl->nfrags; |
273 | pdu_cb->dfrag_idx = skb_shinfo(skb)->nr_frags; |
274 | lro_cb->complete = false; |
275 | } else { |
276 | struct cpl_rx_iscsi_cmp *cpl; |
277 | |
278 | cpl = (struct cpl_rx_iscsi_cmp *)gl->va; |
279 | offset = sizeof(struct cpl_rx_iscsi_cmp); |
280 | pdu_cb->flags |= (PDUCBF_RX_HDR | PDUCBF_RX_STATUS); |
281 | len = be16_to_cpu(cpl->len); |
282 | pdu_cb->hdr = gl->va + offset; |
283 | pdu_cb->hlen = len; |
284 | pdu_cb->hfrag_idx = skb_shinfo(skb)->nr_frags; |
285 | pdu_cb->ddigest = be32_to_cpu(cpl->ulp_crc); |
286 | pdu_cb->pdulen = ntohs(cpl->len); |
287 | |
288 | if (unlikely(gl->nfrags > 1)) |
289 | cxgbit_skcb_flags(skb) = 0; |
290 | |
291 | cxgbit_process_ddpvld(csk: lro_cb->csk, pdu_cb, |
292 | be32_to_cpu(cpl->ddpvld)); |
293 | |
294 | if (pdu_cb->flags & PDUCBF_RX_DATA_DDPD) { |
295 | pdu_cb->flags |= PDUCBF_RX_DDP_CMP; |
296 | pdu_cb->complete = true; |
297 | } else if (pdu_cb->flags & PDUCBF_RX_DATA) { |
298 | pdu_cb->complete = true; |
299 | } |
300 | |
301 | lro_cb->pdu_totallen += pdu_cb->hlen + pdu_cb->dlen; |
302 | lro_cb->complete = true; |
303 | lro_cb->pdu_idx++; |
304 | } |
305 | |
306 | cxgbit_copy_frags(skb, gl, offset); |
307 | |
308 | pdu_cb->frags += gl->nfrags; |
309 | lro_cb->offset += len; |
310 | skb->len += len; |
311 | skb->data_len += len; |
312 | skb->truesize += len; |
313 | } |
314 | |
315 | static struct sk_buff * |
316 | cxgbit_lro_init_skb(struct cxgbit_sock *csk, u8 op, const struct pkt_gl *gl, |
317 | const __be64 *rsp, struct napi_struct *napi) |
318 | { |
319 | struct sk_buff *skb; |
320 | struct cxgbit_lro_cb *lro_cb; |
321 | |
322 | skb = napi_alloc_skb(napi, LRO_SKB_MAX_HEADROOM); |
323 | |
324 | if (unlikely(!skb)) |
325 | return NULL; |
326 | |
327 | memset(skb->data, 0, LRO_SKB_MAX_HEADROOM); |
328 | |
329 | cxgbit_skcb_flags(skb) |= SKCBF_RX_LRO; |
330 | |
331 | lro_cb = cxgbit_skb_lro_cb(skb); |
332 | |
333 | cxgbit_get_csk(csk); |
334 | |
335 | lro_cb->csk = csk; |
336 | |
337 | return skb; |
338 | } |
339 | |
340 | static void cxgbit_queue_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb) |
341 | { |
342 | bool wakeup_thread = false; |
343 | |
344 | spin_lock(lock: &csk->rxq.lock); |
345 | __skb_queue_tail(list: &csk->rxq, newsk: skb); |
346 | if (skb_queue_len(list_: &csk->rxq) == 1) |
347 | wakeup_thread = true; |
348 | spin_unlock(lock: &csk->rxq.lock); |
349 | |
350 | if (wakeup_thread) |
351 | wake_up(&csk->waitq); |
352 | } |
353 | |
354 | static void cxgbit_lro_flush(struct t4_lro_mgr *lro_mgr, struct sk_buff *skb) |
355 | { |
356 | struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); |
357 | struct cxgbit_sock *csk = lro_cb->csk; |
358 | |
359 | csk->lro_skb = NULL; |
360 | |
361 | __skb_unlink(skb, list: &lro_mgr->lroq); |
362 | cxgbit_queue_lro_skb(csk, skb); |
363 | |
364 | cxgbit_put_csk(csk); |
365 | |
366 | lro_mgr->lro_pkts++; |
367 | lro_mgr->lro_session_cnt--; |
368 | } |
369 | |
370 | static void cxgbit_uld_lro_flush(struct t4_lro_mgr *lro_mgr) |
371 | { |
372 | struct sk_buff *skb; |
373 | |
374 | while ((skb = skb_peek(list_: &lro_mgr->lroq))) |
375 | cxgbit_lro_flush(lro_mgr, skb); |
376 | } |
377 | |
378 | static int |
379 | cxgbit_lro_receive(struct cxgbit_sock *csk, u8 op, const __be64 *rsp, |
380 | const struct pkt_gl *gl, struct t4_lro_mgr *lro_mgr, |
381 | struct napi_struct *napi) |
382 | { |
383 | struct sk_buff *skb; |
384 | struct cxgbit_lro_cb *lro_cb; |
385 | |
386 | if (!csk) { |
387 | pr_err("%s: csk NULL, op 0x%x.\n" , __func__, op); |
388 | goto out; |
389 | } |
390 | |
391 | if (csk->lro_skb) |
392 | goto add_packet; |
393 | |
394 | start_lro: |
395 | if (lro_mgr->lro_session_cnt >= MAX_LRO_SESSIONS) { |
396 | cxgbit_uld_lro_flush(lro_mgr); |
397 | goto start_lro; |
398 | } |
399 | |
400 | skb = cxgbit_lro_init_skb(csk, op, gl, rsp, napi); |
401 | if (unlikely(!skb)) |
402 | goto out; |
403 | |
404 | csk->lro_skb = skb; |
405 | |
406 | __skb_queue_tail(list: &lro_mgr->lroq, newsk: skb); |
407 | lro_mgr->lro_session_cnt++; |
408 | |
409 | add_packet: |
410 | skb = csk->lro_skb; |
411 | lro_cb = cxgbit_skb_lro_cb(skb); |
412 | |
413 | if ((gl && (((skb_shinfo(skb)->nr_frags + gl->nfrags) > |
414 | MAX_SKB_FRAGS) || (lro_cb->pdu_totallen >= LRO_FLUSH_LEN_MAX))) || |
415 | (lro_cb->pdu_idx >= MAX_SKB_FRAGS)) { |
416 | cxgbit_lro_flush(lro_mgr, skb); |
417 | goto start_lro; |
418 | } |
419 | |
420 | if (gl) |
421 | cxgbit_lro_add_packet_gl(skb, op, gl); |
422 | else |
423 | cxgbit_lro_add_packet_rsp(skb, op, rsp); |
424 | |
425 | lro_mgr->lro_merged++; |
426 | |
427 | return 0; |
428 | |
429 | out: |
430 | return -1; |
431 | } |
432 | |
433 | static int |
434 | cxgbit_uld_lro_rx_handler(void *hndl, const __be64 *rsp, |
435 | const struct pkt_gl *gl, struct t4_lro_mgr *lro_mgr, |
436 | struct napi_struct *napi) |
437 | { |
438 | struct cxgbit_device *cdev = hndl; |
439 | struct cxgb4_lld_info *lldi = &cdev->lldi; |
440 | struct cpl_tx_data *rpl = NULL; |
441 | struct cxgbit_sock *csk = NULL; |
442 | unsigned int tid = 0; |
443 | struct sk_buff *skb; |
444 | unsigned int op = *(u8 *)rsp; |
445 | bool lro_flush = true; |
446 | |
447 | switch (op) { |
448 | case CPL_ISCSI_HDR: |
449 | case CPL_ISCSI_DATA: |
450 | case CPL_RX_ISCSI_CMP: |
451 | case CPL_RX_ISCSI_DDP: |
452 | case CPL_FW4_ACK: |
453 | lro_flush = false; |
454 | fallthrough; |
455 | case CPL_ABORT_RPL_RSS: |
456 | case CPL_PASS_ESTABLISH: |
457 | case CPL_PEER_CLOSE: |
458 | case CPL_CLOSE_CON_RPL: |
459 | case CPL_ABORT_REQ_RSS: |
460 | case CPL_SET_TCB_RPL: |
461 | case CPL_RX_DATA: |
462 | rpl = gl ? (struct cpl_tx_data *)gl->va : |
463 | (struct cpl_tx_data *)(rsp + 1); |
464 | tid = GET_TID(rpl); |
465 | csk = lookup_tid(lldi->tids, tid); |
466 | break; |
467 | default: |
468 | break; |
469 | } |
470 | |
471 | if (csk && csk->lro_skb && lro_flush) |
472 | cxgbit_lro_flush(lro_mgr, skb: csk->lro_skb); |
473 | |
474 | if (!gl) { |
475 | unsigned int len; |
476 | |
477 | if (op == CPL_RX_ISCSI_DDP) { |
478 | if (!cxgbit_lro_receive(csk, op, rsp, NULL, lro_mgr, |
479 | napi)) |
480 | return 0; |
481 | } |
482 | |
483 | len = 64 - sizeof(struct rsp_ctrl) - 8; |
484 | skb = napi_alloc_skb(napi, length: len); |
485 | if (!skb) |
486 | goto nomem; |
487 | __skb_put(skb, len); |
488 | skb_copy_to_linear_data(skb, from: &rsp[1], len); |
489 | } else { |
490 | if (unlikely(op != *(u8 *)gl->va)) { |
491 | pr_info("? FL 0x%p,RSS%#llx,FL %#llx,len %u.\n" , |
492 | gl->va, be64_to_cpu(*rsp), |
493 | get_unaligned_be64(gl->va), |
494 | gl->tot_len); |
495 | return 0; |
496 | } |
497 | |
498 | if ((op == CPL_ISCSI_HDR) || (op == CPL_ISCSI_DATA) || |
499 | (op == CPL_RX_ISCSI_CMP)) { |
500 | if (!cxgbit_lro_receive(csk, op, rsp, gl, lro_mgr, |
501 | napi)) |
502 | return 0; |
503 | } |
504 | |
505 | #define RX_PULL_LEN 128 |
506 | skb = cxgb4_pktgl_to_skb(gl, RX_PULL_LEN, RX_PULL_LEN); |
507 | if (unlikely(!skb)) |
508 | goto nomem; |
509 | } |
510 | |
511 | rpl = (struct cpl_tx_data *)skb->data; |
512 | op = rpl->ot.opcode; |
513 | cxgbit_skcb_rx_opcode(skb) = op; |
514 | |
515 | pr_debug("cdev %p, opcode 0x%x(0x%x,0x%x), skb %p.\n" , |
516 | cdev, op, rpl->ot.opcode_tid, |
517 | ntohl(rpl->ot.opcode_tid), skb); |
518 | |
519 | if (op < NUM_CPL_CMDS && cxgbit_cplhandlers[op]) { |
520 | cxgbit_cplhandlers[op](cdev, skb); |
521 | } else { |
522 | pr_err("No handler for opcode 0x%x.\n" , op); |
523 | __kfree_skb(skb); |
524 | } |
525 | return 0; |
526 | nomem: |
527 | pr_err("%s OOM bailing out.\n" , __func__); |
528 | return 1; |
529 | } |
530 | |
531 | #ifdef CONFIG_CHELSIO_T4_DCB |
532 | struct cxgbit_dcb_work { |
533 | struct dcb_app_type dcb_app; |
534 | struct work_struct work; |
535 | }; |
536 | |
537 | static void |
538 | cxgbit_update_dcb_priority(struct cxgbit_device *cdev, u8 port_id, |
539 | u8 dcb_priority, u16 port_num) |
540 | { |
541 | struct cxgbit_sock *csk; |
542 | struct sk_buff *skb; |
543 | u16 local_port; |
544 | bool wakeup_thread = false; |
545 | |
546 | spin_lock_bh(lock: &cdev->cskq.lock); |
547 | list_for_each_entry(csk, &cdev->cskq.list, list) { |
548 | if (csk->port_id != port_id) |
549 | continue; |
550 | |
551 | if (csk->com.local_addr.ss_family == AF_INET6) { |
552 | struct sockaddr_in6 *sock_in6; |
553 | |
554 | sock_in6 = (struct sockaddr_in6 *)&csk->com.local_addr; |
555 | local_port = ntohs(sock_in6->sin6_port); |
556 | } else { |
557 | struct sockaddr_in *sock_in; |
558 | |
559 | sock_in = (struct sockaddr_in *)&csk->com.local_addr; |
560 | local_port = ntohs(sock_in->sin_port); |
561 | } |
562 | |
563 | if (local_port != port_num) |
564 | continue; |
565 | |
566 | if (csk->dcb_priority == dcb_priority) |
567 | continue; |
568 | |
569 | skb = alloc_skb(size: 0, GFP_ATOMIC); |
570 | if (!skb) |
571 | continue; |
572 | |
573 | spin_lock(lock: &csk->rxq.lock); |
574 | __skb_queue_tail(list: &csk->rxq, newsk: skb); |
575 | if (skb_queue_len(list_: &csk->rxq) == 1) |
576 | wakeup_thread = true; |
577 | spin_unlock(lock: &csk->rxq.lock); |
578 | |
579 | if (wakeup_thread) { |
580 | wake_up(&csk->waitq); |
581 | wakeup_thread = false; |
582 | } |
583 | } |
584 | spin_unlock_bh(lock: &cdev->cskq.lock); |
585 | } |
586 | |
587 | static void cxgbit_dcb_workfn(struct work_struct *work) |
588 | { |
589 | struct cxgbit_dcb_work *dcb_work; |
590 | struct net_device *ndev; |
591 | struct cxgbit_device *cdev = NULL; |
592 | struct dcb_app_type *iscsi_app; |
593 | u8 priority, port_id = 0xff; |
594 | |
595 | dcb_work = container_of(work, struct cxgbit_dcb_work, work); |
596 | iscsi_app = &dcb_work->dcb_app; |
597 | |
598 | if (iscsi_app->dcbx & DCB_CAP_DCBX_VER_IEEE) { |
599 | if ((iscsi_app->app.selector != IEEE_8021QAZ_APP_SEL_STREAM) && |
600 | (iscsi_app->app.selector != IEEE_8021QAZ_APP_SEL_ANY)) |
601 | goto out; |
602 | |
603 | priority = iscsi_app->app.priority; |
604 | |
605 | } else if (iscsi_app->dcbx & DCB_CAP_DCBX_VER_CEE) { |
606 | if (iscsi_app->app.selector != DCB_APP_IDTYPE_PORTNUM) |
607 | goto out; |
608 | |
609 | if (!iscsi_app->app.priority) |
610 | goto out; |
611 | |
612 | priority = ffs(iscsi_app->app.priority) - 1; |
613 | } else { |
614 | goto out; |
615 | } |
616 | |
617 | pr_debug("priority for ifid %d is %u\n" , |
618 | iscsi_app->ifindex, priority); |
619 | |
620 | ndev = dev_get_by_index(net: &init_net, ifindex: iscsi_app->ifindex); |
621 | |
622 | if (!ndev) |
623 | goto out; |
624 | |
625 | mutex_lock(&cdev_list_lock); |
626 | cdev = cxgbit_find_device(ndev, &port_id); |
627 | |
628 | dev_put(dev: ndev); |
629 | |
630 | if (!cdev) { |
631 | mutex_unlock(lock: &cdev_list_lock); |
632 | goto out; |
633 | } |
634 | |
635 | cxgbit_update_dcb_priority(cdev, port_id, dcb_priority: priority, |
636 | port_num: iscsi_app->app.protocol); |
637 | mutex_unlock(lock: &cdev_list_lock); |
638 | out: |
639 | kfree(objp: dcb_work); |
640 | } |
641 | |
642 | static int |
643 | cxgbit_dcbevent_notify(struct notifier_block *nb, unsigned long action, |
644 | void *data) |
645 | { |
646 | struct cxgbit_dcb_work *dcb_work; |
647 | struct dcb_app_type *dcb_app = data; |
648 | |
649 | dcb_work = kzalloc(size: sizeof(*dcb_work), GFP_ATOMIC); |
650 | if (!dcb_work) |
651 | return NOTIFY_DONE; |
652 | |
653 | dcb_work->dcb_app = *dcb_app; |
654 | INIT_WORK(&dcb_work->work, cxgbit_dcb_workfn); |
655 | schedule_work(work: &dcb_work->work); |
656 | return NOTIFY_OK; |
657 | } |
658 | #endif |
659 | |
660 | static enum target_prot_op cxgbit_get_sup_prot_ops(struct iscsit_conn *conn) |
661 | { |
662 | return TARGET_PROT_NORMAL; |
663 | } |
664 | |
665 | static struct iscsit_transport cxgbit_transport = { |
666 | .name = DRV_NAME, |
667 | .transport_type = ISCSI_CXGBIT, |
668 | .rdma_shutdown = false, |
669 | .priv_size = sizeof(struct cxgbit_cmd), |
670 | .owner = THIS_MODULE, |
671 | .iscsit_setup_np = cxgbit_setup_np, |
672 | .iscsit_accept_np = cxgbit_accept_np, |
673 | .iscsit_free_np = cxgbit_free_np, |
674 | .iscsit_free_conn = cxgbit_free_conn, |
675 | .iscsit_get_login_rx = cxgbit_get_login_rx, |
676 | .iscsit_put_login_tx = cxgbit_put_login_tx, |
677 | .iscsit_immediate_queue = iscsit_immediate_queue, |
678 | .iscsit_response_queue = iscsit_response_queue, |
679 | .iscsit_get_dataout = iscsit_build_r2ts_for_cmd, |
680 | .iscsit_queue_data_in = iscsit_queue_rsp, |
681 | .iscsit_queue_status = iscsit_queue_rsp, |
682 | .iscsit_xmit_pdu = cxgbit_xmit_pdu, |
683 | .iscsit_get_r2t_ttt = cxgbit_get_r2t_ttt, |
684 | .iscsit_get_rx_pdu = cxgbit_get_rx_pdu, |
685 | .iscsit_validate_params = cxgbit_validate_params, |
686 | .iscsit_unmap_cmd = cxgbit_unmap_cmd, |
687 | .iscsit_aborted_task = iscsit_aborted_task, |
688 | .iscsit_get_sup_prot_ops = cxgbit_get_sup_prot_ops, |
689 | }; |
690 | |
691 | static struct cxgb4_uld_info cxgbit_uld_info = { |
692 | .name = DRV_NAME, |
693 | .nrxq = MAX_ULD_QSETS, |
694 | .ntxq = MAX_ULD_QSETS, |
695 | .rxq_size = 1024, |
696 | .lro = true, |
697 | .add = cxgbit_uld_add, |
698 | .state_change = cxgbit_uld_state_change, |
699 | .lro_rx_handler = cxgbit_uld_lro_rx_handler, |
700 | .lro_flush = cxgbit_uld_lro_flush, |
701 | }; |
702 | |
703 | #ifdef CONFIG_CHELSIO_T4_DCB |
704 | static struct notifier_block cxgbit_dcbevent_nb = { |
705 | .notifier_call = cxgbit_dcbevent_notify, |
706 | }; |
707 | #endif |
708 | |
709 | static int __init cxgbit_init(void) |
710 | { |
711 | cxgb4_register_uld(CXGB4_ULD_ISCSIT, &cxgbit_uld_info); |
712 | iscsit_register_transport(&cxgbit_transport); |
713 | |
714 | #ifdef CONFIG_CHELSIO_T4_DCB |
715 | pr_info("%s dcb enabled.\n" , DRV_NAME); |
716 | register_dcbevent_notifier(nb: &cxgbit_dcbevent_nb); |
717 | #endif |
718 | BUILD_BUG_ON(sizeof_field(struct sk_buff, cb) < |
719 | sizeof(union cxgbit_skb_cb)); |
720 | return 0; |
721 | } |
722 | |
723 | static void __exit cxgbit_exit(void) |
724 | { |
725 | struct cxgbit_device *cdev, *tmp; |
726 | |
727 | #ifdef CONFIG_CHELSIO_T4_DCB |
728 | unregister_dcbevent_notifier(nb: &cxgbit_dcbevent_nb); |
729 | #endif |
730 | mutex_lock(&cdev_list_lock); |
731 | list_for_each_entry_safe(cdev, tmp, &cdev_list_head, list) { |
732 | list_del(entry: &cdev->list); |
733 | cxgbit_put_cdev(cdev); |
734 | } |
735 | mutex_unlock(lock: &cdev_list_lock); |
736 | iscsit_unregister_transport(&cxgbit_transport); |
737 | cxgb4_unregister_uld(CXGB4_ULD_ISCSIT); |
738 | } |
739 | |
740 | module_init(cxgbit_init); |
741 | module_exit(cxgbit_exit); |
742 | |
743 | MODULE_DESCRIPTION("Chelsio iSCSI target offload driver" ); |
744 | MODULE_AUTHOR("Chelsio Communications" ); |
745 | MODULE_VERSION(DRV_VERSION); |
746 | MODULE_LICENSE("GPL" ); |
747 | |