1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Shared Memory Communications Direct over ISM devices (SMC-D) |
3 | * |
4 | * Functions for ISM device. |
5 | * |
6 | * Copyright IBM Corp. 2018 |
7 | */ |
8 | |
9 | #include <linux/if_vlan.h> |
10 | #include <linux/spinlock.h> |
11 | #include <linux/mutex.h> |
12 | #include <linux/slab.h> |
13 | #include <asm/page.h> |
14 | |
15 | #include "smc.h" |
16 | #include "smc_core.h" |
17 | #include "smc_ism.h" |
18 | #include "smc_pnet.h" |
19 | #include "smc_netlink.h" |
20 | #include "linux/ism.h" |
21 | |
22 | struct smcd_dev_list smcd_dev_list = { |
23 | .list = LIST_HEAD_INIT(smcd_dev_list.list), |
24 | .mutex = __MUTEX_INITIALIZER(smcd_dev_list.mutex) |
25 | }; |
26 | |
27 | static bool smc_ism_v2_capable; |
28 | static u8 smc_ism_v2_system_eid[SMC_MAX_EID_LEN]; |
29 | |
30 | #if IS_ENABLED(CONFIG_ISM) |
31 | static void smcd_register_dev(struct ism_dev *ism); |
32 | static void smcd_unregister_dev(struct ism_dev *ism); |
33 | static void smcd_handle_event(struct ism_dev *ism, struct ism_event *event); |
34 | static void smcd_handle_irq(struct ism_dev *ism, unsigned int dmbno, |
35 | u16 dmbemask); |
36 | |
37 | static struct ism_client smc_ism_client = { |
38 | .name = "SMC-D" , |
39 | .add = smcd_register_dev, |
40 | .remove = smcd_unregister_dev, |
41 | .handle_event = smcd_handle_event, |
42 | .handle_irq = smcd_handle_irq, |
43 | }; |
44 | #endif |
45 | |
46 | /* Test if an ISM communication is possible - same CPC */ |
47 | int smc_ism_cantalk(u64 peer_gid, unsigned short vlan_id, struct smcd_dev *smcd) |
48 | { |
49 | return smcd->ops->query_remote_gid(smcd, peer_gid, vlan_id ? 1 : 0, |
50 | vlan_id); |
51 | } |
52 | |
53 | void smc_ism_get_system_eid(u8 **eid) |
54 | { |
55 | if (!smc_ism_v2_capable) |
56 | *eid = NULL; |
57 | else |
58 | *eid = smc_ism_v2_system_eid; |
59 | } |
60 | |
61 | u16 smc_ism_get_chid(struct smcd_dev *smcd) |
62 | { |
63 | return smcd->ops->get_chid(smcd); |
64 | } |
65 | |
66 | /* HW supports ISM V2 and thus System EID is defined */ |
67 | bool smc_ism_is_v2_capable(void) |
68 | { |
69 | return smc_ism_v2_capable; |
70 | } |
71 | |
72 | /* Set a connection using this DMBE. */ |
73 | void smc_ism_set_conn(struct smc_connection *conn) |
74 | { |
75 | unsigned long flags; |
76 | |
77 | spin_lock_irqsave(&conn->lgr->smcd->lock, flags); |
78 | conn->lgr->smcd->conn[conn->rmb_desc->sba_idx] = conn; |
79 | spin_unlock_irqrestore(lock: &conn->lgr->smcd->lock, flags); |
80 | } |
81 | |
82 | /* Unset a connection using this DMBE. */ |
83 | void smc_ism_unset_conn(struct smc_connection *conn) |
84 | { |
85 | unsigned long flags; |
86 | |
87 | if (!conn->rmb_desc) |
88 | return; |
89 | |
90 | spin_lock_irqsave(&conn->lgr->smcd->lock, flags); |
91 | conn->lgr->smcd->conn[conn->rmb_desc->sba_idx] = NULL; |
92 | spin_unlock_irqrestore(lock: &conn->lgr->smcd->lock, flags); |
93 | } |
94 | |
95 | /* Register a VLAN identifier with the ISM device. Use a reference count |
96 | * and add a VLAN identifier only when the first DMB using this VLAN is |
97 | * registered. |
98 | */ |
99 | int smc_ism_get_vlan(struct smcd_dev *smcd, unsigned short vlanid) |
100 | { |
101 | struct smc_ism_vlanid *new_vlan, *vlan; |
102 | unsigned long flags; |
103 | int rc = 0; |
104 | |
105 | if (!vlanid) /* No valid vlan id */ |
106 | return -EINVAL; |
107 | |
108 | /* create new vlan entry, in case we need it */ |
109 | new_vlan = kzalloc(size: sizeof(*new_vlan), GFP_KERNEL); |
110 | if (!new_vlan) |
111 | return -ENOMEM; |
112 | new_vlan->vlanid = vlanid; |
113 | refcount_set(r: &new_vlan->refcnt, n: 1); |
114 | |
115 | /* if there is an existing entry, increase count and return */ |
116 | spin_lock_irqsave(&smcd->lock, flags); |
117 | list_for_each_entry(vlan, &smcd->vlan, list) { |
118 | if (vlan->vlanid == vlanid) { |
119 | refcount_inc(r: &vlan->refcnt); |
120 | kfree(objp: new_vlan); |
121 | goto out; |
122 | } |
123 | } |
124 | |
125 | /* no existing entry found. |
126 | * add new entry to device; might fail, e.g., if HW limit reached |
127 | */ |
128 | if (smcd->ops->add_vlan_id(smcd, vlanid)) { |
129 | kfree(objp: new_vlan); |
130 | rc = -EIO; |
131 | goto out; |
132 | } |
133 | list_add_tail(new: &new_vlan->list, head: &smcd->vlan); |
134 | out: |
135 | spin_unlock_irqrestore(lock: &smcd->lock, flags); |
136 | return rc; |
137 | } |
138 | |
139 | /* Unregister a VLAN identifier with the ISM device. Use a reference count |
140 | * and remove a VLAN identifier only when the last DMB using this VLAN is |
141 | * unregistered. |
142 | */ |
143 | int smc_ism_put_vlan(struct smcd_dev *smcd, unsigned short vlanid) |
144 | { |
145 | struct smc_ism_vlanid *vlan; |
146 | unsigned long flags; |
147 | bool found = false; |
148 | int rc = 0; |
149 | |
150 | if (!vlanid) /* No valid vlan id */ |
151 | return -EINVAL; |
152 | |
153 | spin_lock_irqsave(&smcd->lock, flags); |
154 | list_for_each_entry(vlan, &smcd->vlan, list) { |
155 | if (vlan->vlanid == vlanid) { |
156 | if (!refcount_dec_and_test(r: &vlan->refcnt)) |
157 | goto out; |
158 | found = true; |
159 | break; |
160 | } |
161 | } |
162 | if (!found) { |
163 | rc = -ENOENT; |
164 | goto out; /* VLAN id not in table */ |
165 | } |
166 | |
167 | /* Found and the last reference just gone */ |
168 | if (smcd->ops->del_vlan_id(smcd, vlanid)) |
169 | rc = -EIO; |
170 | list_del(entry: &vlan->list); |
171 | kfree(objp: vlan); |
172 | out: |
173 | spin_unlock_irqrestore(lock: &smcd->lock, flags); |
174 | return rc; |
175 | } |
176 | |
177 | int smc_ism_unregister_dmb(struct smcd_dev *smcd, struct smc_buf_desc *dmb_desc) |
178 | { |
179 | struct smcd_dmb dmb; |
180 | int rc = 0; |
181 | |
182 | if (!dmb_desc->dma_addr) |
183 | return rc; |
184 | |
185 | memset(&dmb, 0, sizeof(dmb)); |
186 | dmb.dmb_tok = dmb_desc->token; |
187 | dmb.sba_idx = dmb_desc->sba_idx; |
188 | dmb.cpu_addr = dmb_desc->cpu_addr; |
189 | dmb.dma_addr = dmb_desc->dma_addr; |
190 | dmb.dmb_len = dmb_desc->len; |
191 | rc = smcd->ops->unregister_dmb(smcd, &dmb); |
192 | if (!rc || rc == ISM_ERROR) { |
193 | dmb_desc->cpu_addr = NULL; |
194 | dmb_desc->dma_addr = 0; |
195 | } |
196 | |
197 | return rc; |
198 | } |
199 | |
200 | int smc_ism_register_dmb(struct smc_link_group *lgr, int dmb_len, |
201 | struct smc_buf_desc *dmb_desc) |
202 | { |
203 | #if IS_ENABLED(CONFIG_ISM) |
204 | struct smcd_dmb dmb; |
205 | int rc; |
206 | |
207 | memset(&dmb, 0, sizeof(dmb)); |
208 | dmb.dmb_len = dmb_len; |
209 | dmb.sba_idx = dmb_desc->sba_idx; |
210 | dmb.vlan_id = lgr->vlan_id; |
211 | dmb.rgid = lgr->peer_gid; |
212 | rc = lgr->smcd->ops->register_dmb(lgr->smcd, &dmb, &smc_ism_client); |
213 | if (!rc) { |
214 | dmb_desc->sba_idx = dmb.sba_idx; |
215 | dmb_desc->token = dmb.dmb_tok; |
216 | dmb_desc->cpu_addr = dmb.cpu_addr; |
217 | dmb_desc->dma_addr = dmb.dma_addr; |
218 | dmb_desc->len = dmb.dmb_len; |
219 | } |
220 | return rc; |
221 | #else |
222 | return 0; |
223 | #endif |
224 | } |
225 | |
226 | static int smc_nl_handle_smcd_dev(struct smcd_dev *smcd, |
227 | struct sk_buff *skb, |
228 | struct netlink_callback *cb) |
229 | { |
230 | char smc_pnet[SMC_MAX_PNETID_LEN + 1]; |
231 | struct smc_pci_dev smc_pci_dev; |
232 | struct nlattr *port_attrs; |
233 | struct nlattr *attrs; |
234 | struct ism_dev *ism; |
235 | int use_cnt = 0; |
236 | void *nlh; |
237 | |
238 | ism = smcd->priv; |
239 | nlh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, seq: cb->nlh->nlmsg_seq, |
240 | family: &smc_gen_nl_family, NLM_F_MULTI, |
241 | cmd: SMC_NETLINK_GET_DEV_SMCD); |
242 | if (!nlh) |
243 | goto errmsg; |
244 | attrs = nla_nest_start(skb, attrtype: SMC_GEN_DEV_SMCD); |
245 | if (!attrs) |
246 | goto errout; |
247 | use_cnt = atomic_read(v: &smcd->lgr_cnt); |
248 | if (nla_put_u32(skb, attrtype: SMC_NLA_DEV_USE_CNT, value: use_cnt)) |
249 | goto errattr; |
250 | if (nla_put_u8(skb, attrtype: SMC_NLA_DEV_IS_CRIT, value: use_cnt > 0)) |
251 | goto errattr; |
252 | memset(&smc_pci_dev, 0, sizeof(smc_pci_dev)); |
253 | smc_set_pci_values(to_pci_dev(ism->dev.parent), smc_dev: &smc_pci_dev); |
254 | if (nla_put_u32(skb, attrtype: SMC_NLA_DEV_PCI_FID, value: smc_pci_dev.pci_fid)) |
255 | goto errattr; |
256 | if (nla_put_u16(skb, attrtype: SMC_NLA_DEV_PCI_CHID, value: smc_pci_dev.pci_pchid)) |
257 | goto errattr; |
258 | if (nla_put_u16(skb, attrtype: SMC_NLA_DEV_PCI_VENDOR, value: smc_pci_dev.pci_vendor)) |
259 | goto errattr; |
260 | if (nla_put_u16(skb, attrtype: SMC_NLA_DEV_PCI_DEVICE, value: smc_pci_dev.pci_device)) |
261 | goto errattr; |
262 | if (nla_put_string(skb, attrtype: SMC_NLA_DEV_PCI_ID, str: smc_pci_dev.pci_id)) |
263 | goto errattr; |
264 | |
265 | port_attrs = nla_nest_start(skb, attrtype: SMC_NLA_DEV_PORT); |
266 | if (!port_attrs) |
267 | goto errattr; |
268 | if (nla_put_u8(skb, attrtype: SMC_NLA_DEV_PORT_PNET_USR, value: smcd->pnetid_by_user)) |
269 | goto errportattr; |
270 | memcpy(smc_pnet, smcd->pnetid, SMC_MAX_PNETID_LEN); |
271 | smc_pnet[SMC_MAX_PNETID_LEN] = 0; |
272 | if (nla_put_string(skb, attrtype: SMC_NLA_DEV_PORT_PNETID, str: smc_pnet)) |
273 | goto errportattr; |
274 | |
275 | nla_nest_end(skb, start: port_attrs); |
276 | nla_nest_end(skb, start: attrs); |
277 | genlmsg_end(skb, hdr: nlh); |
278 | return 0; |
279 | |
280 | errportattr: |
281 | nla_nest_cancel(skb, start: port_attrs); |
282 | errattr: |
283 | nla_nest_cancel(skb, start: attrs); |
284 | errout: |
285 | nlmsg_cancel(skb, nlh); |
286 | errmsg: |
287 | return -EMSGSIZE; |
288 | } |
289 | |
290 | static void smc_nl_prep_smcd_dev(struct smcd_dev_list *dev_list, |
291 | struct sk_buff *skb, |
292 | struct netlink_callback *cb) |
293 | { |
294 | struct smc_nl_dmp_ctx *cb_ctx = smc_nl_dmp_ctx(c: cb); |
295 | int snum = cb_ctx->pos[0]; |
296 | struct smcd_dev *smcd; |
297 | int num = 0; |
298 | |
299 | mutex_lock(&dev_list->mutex); |
300 | list_for_each_entry(smcd, &dev_list->list, list) { |
301 | if (num < snum) |
302 | goto next; |
303 | if (smc_nl_handle_smcd_dev(smcd, skb, cb)) |
304 | goto errout; |
305 | next: |
306 | num++; |
307 | } |
308 | errout: |
309 | mutex_unlock(lock: &dev_list->mutex); |
310 | cb_ctx->pos[0] = num; |
311 | } |
312 | |
313 | int smcd_nl_get_device(struct sk_buff *skb, struct netlink_callback *cb) |
314 | { |
315 | smc_nl_prep_smcd_dev(dev_list: &smcd_dev_list, skb, cb); |
316 | return skb->len; |
317 | } |
318 | |
319 | #if IS_ENABLED(CONFIG_ISM) |
320 | struct smc_ism_event_work { |
321 | struct work_struct work; |
322 | struct smcd_dev *smcd; |
323 | struct ism_event event; |
324 | }; |
325 | |
326 | #define ISM_EVENT_REQUEST 0x0001 |
327 | #define ISM_EVENT_RESPONSE 0x0002 |
328 | #define ISM_EVENT_REQUEST_IR 0x00000001 |
329 | #define ISM_EVENT_CODE_SHUTDOWN 0x80 |
330 | #define ISM_EVENT_CODE_TESTLINK 0x83 |
331 | |
332 | union smcd_sw_event_info { |
333 | u64 info; |
334 | struct { |
335 | u8 uid[SMC_LGR_ID_SIZE]; |
336 | unsigned short vlan_id; |
337 | u16 code; |
338 | }; |
339 | }; |
340 | |
341 | static void smcd_handle_sw_event(struct smc_ism_event_work *wrk) |
342 | { |
343 | union smcd_sw_event_info ev_info; |
344 | |
345 | ev_info.info = wrk->event.info; |
346 | switch (wrk->event.code) { |
347 | case ISM_EVENT_CODE_SHUTDOWN: /* Peer shut down DMBs */ |
348 | smc_smcd_terminate(wrk->smcd, wrk->event.tok, ev_info.vlan_id); |
349 | break; |
350 | case ISM_EVENT_CODE_TESTLINK: /* Activity timer */ |
351 | if (ev_info.code == ISM_EVENT_REQUEST) { |
352 | ev_info.code = ISM_EVENT_RESPONSE; |
353 | wrk->smcd->ops->signal_event(wrk->smcd, |
354 | wrk->event.tok, |
355 | ISM_EVENT_REQUEST_IR, |
356 | ISM_EVENT_CODE_TESTLINK, |
357 | ev_info.info); |
358 | } |
359 | break; |
360 | } |
361 | } |
362 | |
363 | /* worker for SMC-D events */ |
364 | static void smc_ism_event_work(struct work_struct *work) |
365 | { |
366 | struct smc_ism_event_work *wrk = |
367 | container_of(work, struct smc_ism_event_work, work); |
368 | |
369 | switch (wrk->event.type) { |
370 | case ISM_EVENT_GID: /* GID event, token is peer GID */ |
371 | smc_smcd_terminate(wrk->smcd, wrk->event.tok, VLAN_VID_MASK); |
372 | break; |
373 | case ISM_EVENT_DMB: |
374 | break; |
375 | case ISM_EVENT_SWR: /* Software defined event */ |
376 | smcd_handle_sw_event(wrk); |
377 | break; |
378 | } |
379 | kfree(wrk); |
380 | } |
381 | |
382 | static struct smcd_dev *smcd_alloc_dev(struct device *parent, const char *name, |
383 | const struct smcd_ops *ops, int max_dmbs) |
384 | { |
385 | struct smcd_dev *smcd; |
386 | |
387 | smcd = devm_kzalloc(parent, sizeof(*smcd), GFP_KERNEL); |
388 | if (!smcd) |
389 | return NULL; |
390 | smcd->conn = devm_kcalloc(parent, max_dmbs, |
391 | sizeof(struct smc_connection *), GFP_KERNEL); |
392 | if (!smcd->conn) |
393 | return NULL; |
394 | |
395 | smcd->event_wq = alloc_ordered_workqueue("ism_evt_wq-%s)" , |
396 | WQ_MEM_RECLAIM, name); |
397 | if (!smcd->event_wq) |
398 | return NULL; |
399 | |
400 | smcd->ops = ops; |
401 | |
402 | spin_lock_init(&smcd->lock); |
403 | spin_lock_init(&smcd->lgr_lock); |
404 | INIT_LIST_HEAD(&smcd->vlan); |
405 | INIT_LIST_HEAD(&smcd->lgr_list); |
406 | init_waitqueue_head(&smcd->lgrs_deleted); |
407 | return smcd; |
408 | } |
409 | |
410 | static void smcd_register_dev(struct ism_dev *ism) |
411 | { |
412 | const struct smcd_ops *ops = ism_get_smcd_ops(); |
413 | struct smcd_dev *smcd; |
414 | |
415 | if (!ops) |
416 | return; |
417 | |
418 | smcd = smcd_alloc_dev(&ism->pdev->dev, dev_name(&ism->pdev->dev), ops, |
419 | ISM_NR_DMBS); |
420 | if (!smcd) |
421 | return; |
422 | smcd->priv = ism; |
423 | ism_set_priv(ism, &smc_ism_client, smcd); |
424 | if (smc_pnetid_by_dev_port(&ism->pdev->dev, 0, smcd->pnetid)) |
425 | smc_pnetid_by_table_smcd(smcd); |
426 | |
427 | mutex_lock(&smcd_dev_list.mutex); |
428 | if (list_empty(&smcd_dev_list.list)) { |
429 | u8 *system_eid = NULL; |
430 | |
431 | system_eid = smcd->ops->get_system_eid(); |
432 | if (smcd->ops->supports_v2()) { |
433 | smc_ism_v2_capable = true; |
434 | memcpy(smc_ism_v2_system_eid, system_eid, |
435 | SMC_MAX_EID_LEN); |
436 | } |
437 | } |
438 | /* sort list: devices without pnetid before devices with pnetid */ |
439 | if (smcd->pnetid[0]) |
440 | list_add_tail(&smcd->list, &smcd_dev_list.list); |
441 | else |
442 | list_add(&smcd->list, &smcd_dev_list.list); |
443 | mutex_unlock(&smcd_dev_list.mutex); |
444 | |
445 | pr_warn_ratelimited("smc: adding smcd device %s with pnetid %.16s%s\n" , |
446 | dev_name(&ism->dev), smcd->pnetid, |
447 | smcd->pnetid_by_user ? " (user defined)" : "" ); |
448 | |
449 | return; |
450 | } |
451 | |
452 | static void smcd_unregister_dev(struct ism_dev *ism) |
453 | { |
454 | struct smcd_dev *smcd = ism_get_priv(ism, &smc_ism_client); |
455 | |
456 | pr_warn_ratelimited("smc: removing smcd device %s\n" , |
457 | dev_name(&ism->dev)); |
458 | smcd->going_away = 1; |
459 | smc_smcd_terminate_all(smcd); |
460 | mutex_lock(&smcd_dev_list.mutex); |
461 | list_del_init(&smcd->list); |
462 | mutex_unlock(&smcd_dev_list.mutex); |
463 | destroy_workqueue(smcd->event_wq); |
464 | } |
465 | |
466 | /* SMCD Device event handler. Called from ISM device interrupt handler. |
467 | * Parameters are ism device pointer, |
468 | * - event->type (0 --> DMB, 1 --> GID), |
469 | * - event->code (event code), |
470 | * - event->tok (either DMB token when event type 0, or GID when event type 1) |
471 | * - event->time (time of day) |
472 | * - event->info (debug info). |
473 | * |
474 | * Context: |
475 | * - Function called in IRQ context from ISM device driver event handler. |
476 | */ |
477 | static void smcd_handle_event(struct ism_dev *ism, struct ism_event *event) |
478 | { |
479 | struct smcd_dev *smcd = ism_get_priv(ism, &smc_ism_client); |
480 | struct smc_ism_event_work *wrk; |
481 | |
482 | if (smcd->going_away) |
483 | return; |
484 | /* copy event to event work queue, and let it be handled there */ |
485 | wrk = kmalloc(sizeof(*wrk), GFP_ATOMIC); |
486 | if (!wrk) |
487 | return; |
488 | INIT_WORK(&wrk->work, smc_ism_event_work); |
489 | wrk->smcd = smcd; |
490 | wrk->event = *event; |
491 | queue_work(smcd->event_wq, &wrk->work); |
492 | } |
493 | |
494 | /* SMCD Device interrupt handler. Called from ISM device interrupt handler. |
495 | * Parameters are the ism device pointer, DMB number, and the DMBE bitmask. |
496 | * Find the connection and schedule the tasklet for this connection. |
497 | * |
498 | * Context: |
499 | * - Function called in IRQ context from ISM device driver IRQ handler. |
500 | */ |
501 | static void smcd_handle_irq(struct ism_dev *ism, unsigned int dmbno, |
502 | u16 dmbemask) |
503 | { |
504 | struct smcd_dev *smcd = ism_get_priv(ism, &smc_ism_client); |
505 | struct smc_connection *conn = NULL; |
506 | unsigned long flags; |
507 | |
508 | spin_lock_irqsave(&smcd->lock, flags); |
509 | conn = smcd->conn[dmbno]; |
510 | if (conn && !conn->killed) |
511 | tasklet_schedule(&conn->rx_tsklet); |
512 | spin_unlock_irqrestore(&smcd->lock, flags); |
513 | } |
514 | #endif |
515 | |
516 | int smc_ism_signal_shutdown(struct smc_link_group *lgr) |
517 | { |
518 | int rc = 0; |
519 | #if IS_ENABLED(CONFIG_ISM) |
520 | union smcd_sw_event_info ev_info; |
521 | |
522 | if (lgr->peer_shutdown) |
523 | return 0; |
524 | |
525 | memcpy(ev_info.uid, lgr->id, SMC_LGR_ID_SIZE); |
526 | ev_info.vlan_id = lgr->vlan_id; |
527 | ev_info.code = ISM_EVENT_REQUEST; |
528 | rc = lgr->smcd->ops->signal_event(lgr->smcd, lgr->peer_gid, |
529 | ISM_EVENT_REQUEST_IR, |
530 | ISM_EVENT_CODE_SHUTDOWN, |
531 | ev_info.info); |
532 | #endif |
533 | return rc; |
534 | } |
535 | |
536 | int smc_ism_init(void) |
537 | { |
538 | int rc = 0; |
539 | |
540 | #if IS_ENABLED(CONFIG_ISM) |
541 | smc_ism_v2_capable = false; |
542 | memset(smc_ism_v2_system_eid, 0, SMC_MAX_EID_LEN); |
543 | |
544 | rc = ism_register_client(&smc_ism_client); |
545 | #endif |
546 | return rc; |
547 | } |
548 | |
549 | void smc_ism_exit(void) |
550 | { |
551 | #if IS_ENABLED(CONFIG_ISM) |
552 | ism_unregister_client(&smc_ism_client); |
553 | #endif |
554 | } |
555 | |