1 | // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause |
2 | |
3 | /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */ |
4 | /* Copyright (c) 2008-2019, IBM Corporation */ |
5 | |
6 | #include <linux/init.h> |
7 | #include <linux/errno.h> |
8 | #include <linux/netdevice.h> |
9 | #include <linux/inetdevice.h> |
10 | #include <net/net_namespace.h> |
11 | #include <linux/rtnetlink.h> |
12 | #include <linux/if_arp.h> |
13 | #include <linux/list.h> |
14 | #include <linux/kernel.h> |
15 | #include <linux/sched.h> |
16 | #include <linux/module.h> |
17 | #include <linux/dma-mapping.h> |
18 | |
19 | #include <net/addrconf.h> |
20 | #include <rdma/ib_verbs.h> |
21 | #include <rdma/ib_user_verbs.h> |
22 | #include <rdma/rdma_netlink.h> |
23 | #include <linux/kthread.h> |
24 | |
25 | #include "siw.h" |
26 | #include "siw_verbs.h" |
27 | |
28 | MODULE_AUTHOR("Bernard Metzler" ); |
29 | MODULE_DESCRIPTION("Software iWARP Driver" ); |
30 | MODULE_LICENSE("Dual BSD/GPL" ); |
31 | |
32 | /* transmit from user buffer, if possible */ |
33 | const bool zcopy_tx = true; |
34 | |
35 | /* Restrict usage of GSO, if hardware peer iwarp is unable to process |
36 | * large packets. try_gso = true lets siw try to use local GSO, |
37 | * if peer agrees. Not using GSO severly limits siw maximum tx bandwidth. |
38 | */ |
39 | const bool try_gso; |
40 | |
41 | /* Attach siw also with loopback devices */ |
42 | const bool loopback_enabled = true; |
43 | |
44 | /* We try to negotiate CRC on, if true */ |
45 | const bool mpa_crc_required; |
46 | |
47 | /* MPA CRC on/off enforced */ |
48 | const bool mpa_crc_strict; |
49 | |
50 | /* Control TCP_NODELAY socket option */ |
51 | const bool siw_tcp_nagle; |
52 | |
53 | /* Select MPA version to be used during connection setup */ |
54 | u_char mpa_version = MPA_REVISION_2; |
55 | |
56 | /* Selects MPA P2P mode (additional handshake during connection |
57 | * setup, if true. |
58 | */ |
59 | const bool peer_to_peer; |
60 | |
61 | struct task_struct *siw_tx_thread[NR_CPUS]; |
62 | struct crypto_shash *siw_crypto_shash; |
63 | |
64 | static int siw_device_register(struct siw_device *sdev, const char *name) |
65 | { |
66 | struct ib_device *base_dev = &sdev->base_dev; |
67 | static int dev_id = 1; |
68 | int rv; |
69 | |
70 | sdev->vendor_part_id = dev_id++; |
71 | |
72 | rv = ib_register_device(device: base_dev, name, NULL); |
73 | if (rv) { |
74 | pr_warn("siw: device registration error %d\n" , rv); |
75 | return rv; |
76 | } |
77 | |
78 | siw_dbg(base_dev, "HWaddr=%pM\n" , sdev->raw_gid); |
79 | return 0; |
80 | } |
81 | |
82 | static void siw_device_cleanup(struct ib_device *base_dev) |
83 | { |
84 | struct siw_device *sdev = to_siw_dev(base_dev); |
85 | |
86 | xa_destroy(&sdev->qp_xa); |
87 | xa_destroy(&sdev->mem_xa); |
88 | } |
89 | |
90 | static int siw_dev_qualified(struct net_device *netdev) |
91 | { |
92 | /* |
93 | * Additional hardware support can be added here |
94 | * (e.g. ARPHRD_FDDI, ARPHRD_ATM, ...) - see |
95 | * <linux/if_arp.h> for type identifiers. |
96 | */ |
97 | if (netdev->type == ARPHRD_ETHER || netdev->type == ARPHRD_IEEE802 || |
98 | netdev->type == ARPHRD_NONE || |
99 | (netdev->type == ARPHRD_LOOPBACK && loopback_enabled)) |
100 | return 1; |
101 | |
102 | return 0; |
103 | } |
104 | |
105 | static DEFINE_PER_CPU(atomic_t, siw_use_cnt); |
106 | |
107 | static struct { |
108 | struct cpumask **tx_valid_cpus; |
109 | int num_nodes; |
110 | } siw_cpu_info; |
111 | |
112 | static void siw_destroy_cpulist(int number) |
113 | { |
114 | int i = 0; |
115 | |
116 | while (i < number) |
117 | kfree(objp: siw_cpu_info.tx_valid_cpus[i++]); |
118 | |
119 | kfree(objp: siw_cpu_info.tx_valid_cpus); |
120 | siw_cpu_info.tx_valid_cpus = NULL; |
121 | } |
122 | |
123 | static int siw_init_cpulist(void) |
124 | { |
125 | int i, num_nodes = nr_node_ids; |
126 | |
127 | memset(siw_tx_thread, 0, sizeof(siw_tx_thread)); |
128 | |
129 | siw_cpu_info.num_nodes = num_nodes; |
130 | |
131 | siw_cpu_info.tx_valid_cpus = |
132 | kcalloc(n: num_nodes, size: sizeof(struct cpumask *), GFP_KERNEL); |
133 | if (!siw_cpu_info.tx_valid_cpus) { |
134 | siw_cpu_info.num_nodes = 0; |
135 | return -ENOMEM; |
136 | } |
137 | for (i = 0; i < siw_cpu_info.num_nodes; i++) { |
138 | siw_cpu_info.tx_valid_cpus[i] = |
139 | kzalloc(size: sizeof(struct cpumask), GFP_KERNEL); |
140 | if (!siw_cpu_info.tx_valid_cpus[i]) |
141 | goto out_err; |
142 | |
143 | cpumask_clear(dstp: siw_cpu_info.tx_valid_cpus[i]); |
144 | } |
145 | for_each_possible_cpu(i) |
146 | cpumask_set_cpu(cpu: i, dstp: siw_cpu_info.tx_valid_cpus[cpu_to_node(cpu: i)]); |
147 | |
148 | return 0; |
149 | |
150 | out_err: |
151 | siw_cpu_info.num_nodes = 0; |
152 | siw_destroy_cpulist(number: i); |
153 | |
154 | return -ENOMEM; |
155 | } |
156 | |
157 | /* |
158 | * Choose CPU with least number of active QP's from NUMA node of |
159 | * TX interface. |
160 | */ |
161 | int siw_get_tx_cpu(struct siw_device *sdev) |
162 | { |
163 | const struct cpumask *tx_cpumask; |
164 | int i, num_cpus, cpu, min_use, node = sdev->numa_node, tx_cpu = -1; |
165 | |
166 | if (node < 0) |
167 | tx_cpumask = cpu_online_mask; |
168 | else |
169 | tx_cpumask = siw_cpu_info.tx_valid_cpus[node]; |
170 | |
171 | num_cpus = cpumask_weight(srcp: tx_cpumask); |
172 | if (!num_cpus) { |
173 | /* no CPU on this NUMA node */ |
174 | tx_cpumask = cpu_online_mask; |
175 | num_cpus = cpumask_weight(srcp: tx_cpumask); |
176 | } |
177 | if (!num_cpus) |
178 | goto out; |
179 | |
180 | cpu = cpumask_first(srcp: tx_cpumask); |
181 | |
182 | for (i = 0, min_use = SIW_MAX_QP; i < num_cpus; |
183 | i++, cpu = cpumask_next(n: cpu, srcp: tx_cpumask)) { |
184 | int usage; |
185 | |
186 | /* Skip any cores which have no TX thread */ |
187 | if (!siw_tx_thread[cpu]) |
188 | continue; |
189 | |
190 | usage = atomic_read(v: &per_cpu(siw_use_cnt, cpu)); |
191 | if (usage <= min_use) { |
192 | tx_cpu = cpu; |
193 | min_use = usage; |
194 | } |
195 | } |
196 | siw_dbg(&sdev->base_dev, |
197 | "tx cpu %d, node %d, %d qp's\n" , tx_cpu, node, min_use); |
198 | |
199 | out: |
200 | if (tx_cpu >= 0) |
201 | atomic_inc(v: &per_cpu(siw_use_cnt, tx_cpu)); |
202 | else |
203 | pr_warn("siw: no tx cpu found\n" ); |
204 | |
205 | return tx_cpu; |
206 | } |
207 | |
208 | void siw_put_tx_cpu(int cpu) |
209 | { |
210 | atomic_dec(v: &per_cpu(siw_use_cnt, cpu)); |
211 | } |
212 | |
213 | static struct ib_qp *siw_get_base_qp(struct ib_device *base_dev, int id) |
214 | { |
215 | struct siw_qp *qp = siw_qp_id2obj(sdev: to_siw_dev(base_dev), id); |
216 | |
217 | if (qp) { |
218 | /* |
219 | * siw_qp_id2obj() increments object reference count |
220 | */ |
221 | siw_qp_put(qp); |
222 | return &qp->base_qp; |
223 | } |
224 | return NULL; |
225 | } |
226 | |
227 | static const struct ib_device_ops siw_device_ops = { |
228 | .owner = THIS_MODULE, |
229 | .uverbs_abi_ver = SIW_ABI_VERSION, |
230 | .driver_id = RDMA_DRIVER_SIW, |
231 | |
232 | .alloc_mr = siw_alloc_mr, |
233 | .alloc_pd = siw_alloc_pd, |
234 | .alloc_ucontext = siw_alloc_ucontext, |
235 | .create_cq = siw_create_cq, |
236 | .create_qp = siw_create_qp, |
237 | .create_srq = siw_create_srq, |
238 | .dealloc_driver = siw_device_cleanup, |
239 | .dealloc_pd = siw_dealloc_pd, |
240 | .dealloc_ucontext = siw_dealloc_ucontext, |
241 | .dereg_mr = siw_dereg_mr, |
242 | .destroy_cq = siw_destroy_cq, |
243 | .destroy_qp = siw_destroy_qp, |
244 | .destroy_srq = siw_destroy_srq, |
245 | .get_dma_mr = siw_get_dma_mr, |
246 | .get_port_immutable = siw_get_port_immutable, |
247 | .iw_accept = siw_accept, |
248 | .iw_add_ref = siw_qp_get_ref, |
249 | .iw_connect = siw_connect, |
250 | .iw_create_listen = siw_create_listen, |
251 | .iw_destroy_listen = siw_destroy_listen, |
252 | .iw_get_qp = siw_get_base_qp, |
253 | .iw_reject = siw_reject, |
254 | .iw_rem_ref = siw_qp_put_ref, |
255 | .map_mr_sg = siw_map_mr_sg, |
256 | .mmap = siw_mmap, |
257 | .mmap_free = siw_mmap_free, |
258 | .modify_qp = siw_verbs_modify_qp, |
259 | .modify_srq = siw_modify_srq, |
260 | .poll_cq = siw_poll_cq, |
261 | .post_recv = siw_post_receive, |
262 | .post_send = siw_post_send, |
263 | .post_srq_recv = siw_post_srq_recv, |
264 | .query_device = siw_query_device, |
265 | .query_gid = siw_query_gid, |
266 | .query_port = siw_query_port, |
267 | .query_qp = siw_query_qp, |
268 | .query_srq = siw_query_srq, |
269 | .req_notify_cq = siw_req_notify_cq, |
270 | .reg_user_mr = siw_reg_user_mr, |
271 | |
272 | INIT_RDMA_OBJ_SIZE(ib_cq, siw_cq, base_cq), |
273 | INIT_RDMA_OBJ_SIZE(ib_pd, siw_pd, base_pd), |
274 | INIT_RDMA_OBJ_SIZE(ib_qp, siw_qp, base_qp), |
275 | INIT_RDMA_OBJ_SIZE(ib_srq, siw_srq, base_srq), |
276 | INIT_RDMA_OBJ_SIZE(ib_ucontext, siw_ucontext, base_ucontext), |
277 | }; |
278 | |
279 | static struct siw_device *siw_device_create(struct net_device *netdev) |
280 | { |
281 | struct siw_device *sdev = NULL; |
282 | struct ib_device *base_dev; |
283 | int rv; |
284 | |
285 | sdev = ib_alloc_device(siw_device, base_dev); |
286 | if (!sdev) |
287 | return NULL; |
288 | |
289 | base_dev = &sdev->base_dev; |
290 | sdev->netdev = netdev; |
291 | |
292 | if (netdev->addr_len) { |
293 | memcpy(sdev->raw_gid, netdev->dev_addr, |
294 | min_t(unsigned int, netdev->addr_len, ETH_ALEN)); |
295 | } else { |
296 | /* |
297 | * This device does not have a HW address, but |
298 | * connection mangagement requires a unique gid. |
299 | */ |
300 | eth_random_addr(addr: sdev->raw_gid); |
301 | } |
302 | addrconf_addr_eui48(eui: (u8 *)&base_dev->node_guid, addr: sdev->raw_gid); |
303 | |
304 | base_dev->uverbs_cmd_mask |= BIT_ULL(IB_USER_VERBS_CMD_POST_SEND); |
305 | |
306 | base_dev->node_type = RDMA_NODE_RNIC; |
307 | memcpy(base_dev->node_desc, SIW_NODE_DESC_COMMON, |
308 | sizeof(SIW_NODE_DESC_COMMON)); |
309 | |
310 | /* |
311 | * Current model (one-to-one device association): |
312 | * One Softiwarp device per net_device or, equivalently, |
313 | * per physical port. |
314 | */ |
315 | base_dev->phys_port_cnt = 1; |
316 | base_dev->num_comp_vectors = num_possible_cpus(); |
317 | |
318 | xa_init_flags(xa: &sdev->qp_xa, XA_FLAGS_ALLOC1); |
319 | xa_init_flags(xa: &sdev->mem_xa, XA_FLAGS_ALLOC1); |
320 | |
321 | ib_set_device_ops(device: base_dev, ops: &siw_device_ops); |
322 | rv = ib_device_set_netdev(ib_dev: base_dev, ndev: netdev, port: 1); |
323 | if (rv) |
324 | goto error; |
325 | |
326 | memcpy(base_dev->iw_ifname, netdev->name, |
327 | sizeof(base_dev->iw_ifname)); |
328 | |
329 | /* Disable TCP port mapping */ |
330 | base_dev->iw_driver_flags = IW_F_NO_PORT_MAP; |
331 | |
332 | sdev->attrs.max_qp = SIW_MAX_QP; |
333 | sdev->attrs.max_qp_wr = SIW_MAX_QP_WR; |
334 | sdev->attrs.max_ord = SIW_MAX_ORD_QP; |
335 | sdev->attrs.max_ird = SIW_MAX_IRD_QP; |
336 | sdev->attrs.max_sge = SIW_MAX_SGE; |
337 | sdev->attrs.max_sge_rd = SIW_MAX_SGE_RD; |
338 | sdev->attrs.max_cq = SIW_MAX_CQ; |
339 | sdev->attrs.max_cqe = SIW_MAX_CQE; |
340 | sdev->attrs.max_mr = SIW_MAX_MR; |
341 | sdev->attrs.max_pd = SIW_MAX_PD; |
342 | sdev->attrs.max_mw = SIW_MAX_MW; |
343 | sdev->attrs.max_srq = SIW_MAX_SRQ; |
344 | sdev->attrs.max_srq_wr = SIW_MAX_SRQ_WR; |
345 | sdev->attrs.max_srq_sge = SIW_MAX_SGE; |
346 | |
347 | INIT_LIST_HEAD(list: &sdev->cep_list); |
348 | INIT_LIST_HEAD(list: &sdev->qp_list); |
349 | |
350 | atomic_set(v: &sdev->num_ctx, i: 0); |
351 | atomic_set(v: &sdev->num_srq, i: 0); |
352 | atomic_set(v: &sdev->num_qp, i: 0); |
353 | atomic_set(v: &sdev->num_cq, i: 0); |
354 | atomic_set(v: &sdev->num_mr, i: 0); |
355 | atomic_set(v: &sdev->num_pd, i: 0); |
356 | |
357 | sdev->numa_node = dev_to_node(dev: &netdev->dev); |
358 | spin_lock_init(&sdev->lock); |
359 | |
360 | return sdev; |
361 | error: |
362 | ib_dealloc_device(device: base_dev); |
363 | |
364 | return NULL; |
365 | } |
366 | |
367 | /* |
368 | * Network link becomes unavailable. Mark all |
369 | * affected QP's accordingly. |
370 | */ |
371 | static void siw_netdev_down(struct work_struct *work) |
372 | { |
373 | struct siw_device *sdev = |
374 | container_of(work, struct siw_device, netdev_down); |
375 | |
376 | struct siw_qp_attrs qp_attrs; |
377 | struct list_head *pos, *tmp; |
378 | |
379 | memset(&qp_attrs, 0, sizeof(qp_attrs)); |
380 | qp_attrs.state = SIW_QP_STATE_ERROR; |
381 | |
382 | list_for_each_safe(pos, tmp, &sdev->qp_list) { |
383 | struct siw_qp *qp = list_entry(pos, struct siw_qp, devq); |
384 | |
385 | down_write(sem: &qp->state_lock); |
386 | WARN_ON(siw_qp_modify(qp, &qp_attrs, SIW_QP_ATTR_STATE)); |
387 | up_write(sem: &qp->state_lock); |
388 | } |
389 | ib_device_put(device: &sdev->base_dev); |
390 | } |
391 | |
392 | static void siw_device_goes_down(struct siw_device *sdev) |
393 | { |
394 | if (ib_device_try_get(dev: &sdev->base_dev)) { |
395 | INIT_WORK(&sdev->netdev_down, siw_netdev_down); |
396 | schedule_work(work: &sdev->netdev_down); |
397 | } |
398 | } |
399 | |
400 | static int siw_netdev_event(struct notifier_block *nb, unsigned long event, |
401 | void *arg) |
402 | { |
403 | struct net_device *netdev = netdev_notifier_info_to_dev(info: arg); |
404 | struct ib_device *base_dev; |
405 | struct siw_device *sdev; |
406 | |
407 | dev_dbg(&netdev->dev, "siw: event %lu\n" , event); |
408 | |
409 | base_dev = ib_device_get_by_netdev(ndev: netdev, driver_id: RDMA_DRIVER_SIW); |
410 | if (!base_dev) |
411 | return NOTIFY_OK; |
412 | |
413 | sdev = to_siw_dev(base_dev); |
414 | |
415 | switch (event) { |
416 | case NETDEV_UP: |
417 | sdev->state = IB_PORT_ACTIVE; |
418 | siw_port_event(dev: sdev, port: 1, type: IB_EVENT_PORT_ACTIVE); |
419 | break; |
420 | |
421 | case NETDEV_GOING_DOWN: |
422 | siw_device_goes_down(sdev); |
423 | break; |
424 | |
425 | case NETDEV_DOWN: |
426 | sdev->state = IB_PORT_DOWN; |
427 | siw_port_event(dev: sdev, port: 1, type: IB_EVENT_PORT_ERR); |
428 | break; |
429 | |
430 | case NETDEV_REGISTER: |
431 | /* |
432 | * Device registration now handled only by |
433 | * rdma netlink commands. So it shall be impossible |
434 | * to end up here with a valid siw device. |
435 | */ |
436 | siw_dbg(base_dev, "unexpected NETDEV_REGISTER event\n" ); |
437 | break; |
438 | |
439 | case NETDEV_UNREGISTER: |
440 | ib_unregister_device_queued(ib_dev: &sdev->base_dev); |
441 | break; |
442 | |
443 | case NETDEV_CHANGEADDR: |
444 | siw_port_event(dev: sdev, port: 1, type: IB_EVENT_LID_CHANGE); |
445 | break; |
446 | /* |
447 | * Todo: Below netdev events are currently not handled. |
448 | */ |
449 | case NETDEV_CHANGEMTU: |
450 | case NETDEV_CHANGE: |
451 | break; |
452 | |
453 | default: |
454 | break; |
455 | } |
456 | ib_device_put(device: &sdev->base_dev); |
457 | |
458 | return NOTIFY_OK; |
459 | } |
460 | |
461 | static struct notifier_block siw_netdev_nb = { |
462 | .notifier_call = siw_netdev_event, |
463 | }; |
464 | |
465 | static int siw_newlink(const char *basedev_name, struct net_device *netdev) |
466 | { |
467 | struct ib_device *base_dev; |
468 | struct siw_device *sdev = NULL; |
469 | int rv = -ENOMEM; |
470 | |
471 | if (!siw_dev_qualified(netdev)) |
472 | return -EINVAL; |
473 | |
474 | base_dev = ib_device_get_by_netdev(ndev: netdev, driver_id: RDMA_DRIVER_SIW); |
475 | if (base_dev) { |
476 | ib_device_put(device: base_dev); |
477 | return -EEXIST; |
478 | } |
479 | sdev = siw_device_create(netdev); |
480 | if (sdev) { |
481 | dev_dbg(&netdev->dev, "siw: new device\n" ); |
482 | |
483 | if (netif_running(dev: netdev) && netif_carrier_ok(dev: netdev)) |
484 | sdev->state = IB_PORT_ACTIVE; |
485 | else |
486 | sdev->state = IB_PORT_DOWN; |
487 | |
488 | rv = siw_device_register(sdev, name: basedev_name); |
489 | if (rv) |
490 | ib_dealloc_device(device: &sdev->base_dev); |
491 | } |
492 | return rv; |
493 | } |
494 | |
495 | static struct rdma_link_ops siw_link_ops = { |
496 | .type = "siw" , |
497 | .newlink = siw_newlink, |
498 | }; |
499 | |
500 | /* |
501 | * siw_init_module - Initialize Softiwarp module and register with netdev |
502 | * subsystem. |
503 | */ |
504 | static __init int siw_init_module(void) |
505 | { |
506 | int rv; |
507 | |
508 | if (SENDPAGE_THRESH < SIW_MAX_INLINE) { |
509 | pr_info("siw: sendpage threshold too small: %u\n" , |
510 | (int)SENDPAGE_THRESH); |
511 | rv = -EINVAL; |
512 | goto out_error; |
513 | } |
514 | rv = siw_init_cpulist(); |
515 | if (rv) |
516 | goto out_error; |
517 | |
518 | rv = siw_cm_init(); |
519 | if (rv) |
520 | goto out_error; |
521 | |
522 | if (!siw_create_tx_threads()) { |
523 | pr_info("siw: Could not start any TX thread\n" ); |
524 | rv = -ENOMEM; |
525 | goto out_error; |
526 | } |
527 | /* |
528 | * Locate CRC32 algorithm. If unsuccessful, fail |
529 | * loading siw only, if CRC is required. |
530 | */ |
531 | siw_crypto_shash = crypto_alloc_shash(alg_name: "crc32c" , type: 0, mask: 0); |
532 | if (IS_ERR(ptr: siw_crypto_shash)) { |
533 | pr_info("siw: Loading CRC32c failed: %ld\n" , |
534 | PTR_ERR(siw_crypto_shash)); |
535 | siw_crypto_shash = NULL; |
536 | if (mpa_crc_required) { |
537 | rv = -EOPNOTSUPP; |
538 | goto out_error; |
539 | } |
540 | } |
541 | rv = register_netdevice_notifier(nb: &siw_netdev_nb); |
542 | if (rv) |
543 | goto out_error; |
544 | |
545 | rdma_link_register(ops: &siw_link_ops); |
546 | |
547 | pr_info("SoftiWARP attached\n" ); |
548 | return 0; |
549 | |
550 | out_error: |
551 | siw_stop_tx_threads(); |
552 | |
553 | if (siw_crypto_shash) |
554 | crypto_free_shash(tfm: siw_crypto_shash); |
555 | |
556 | pr_info("SoftIWARP attach failed. Error: %d\n" , rv); |
557 | |
558 | siw_cm_exit(); |
559 | siw_destroy_cpulist(number: siw_cpu_info.num_nodes); |
560 | |
561 | return rv; |
562 | } |
563 | |
564 | static void __exit siw_exit_module(void) |
565 | { |
566 | siw_stop_tx_threads(); |
567 | |
568 | unregister_netdevice_notifier(nb: &siw_netdev_nb); |
569 | rdma_link_unregister(ops: &siw_link_ops); |
570 | ib_unregister_driver(driver_id: RDMA_DRIVER_SIW); |
571 | |
572 | siw_cm_exit(); |
573 | |
574 | siw_destroy_cpulist(number: siw_cpu_info.num_nodes); |
575 | |
576 | if (siw_crypto_shash) |
577 | crypto_free_shash(tfm: siw_crypto_shash); |
578 | |
579 | pr_info("SoftiWARP detached\n" ); |
580 | } |
581 | |
582 | module_init(siw_init_module); |
583 | module_exit(siw_exit_module); |
584 | |
585 | MODULE_ALIAS_RDMA_LINK("siw" ); |
586 | |