1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * Copyright 2014 IBM Corp. |
4 | */ |
5 | |
6 | #include <linux/pci.h> |
7 | #include <linux/slab.h> |
8 | #include <linux/file.h> |
9 | #include <misc/cxl.h> |
10 | #include <linux/module.h> |
11 | #include <linux/mount.h> |
12 | #include <linux/pseudo_fs.h> |
13 | #include <linux/sched/mm.h> |
14 | #include <linux/mmu_context.h> |
15 | #include <linux/irqdomain.h> |
16 | |
17 | #include "cxl.h" |
18 | |
19 | /* |
20 | * Since we want to track memory mappings to be able to force-unmap |
21 | * when the AFU is no longer reachable, we need an inode. For devices |
22 | * opened through the cxl user API, this is not a problem, but a |
23 | * userland process can also get a cxl fd through the cxl_get_fd() |
24 | * API, which is used by the cxlflash driver. |
25 | * |
26 | * Therefore we implement our own simple pseudo-filesystem and inode |
27 | * allocator. We don't use the anonymous inode, as we need the |
28 | * meta-data associated with it (address_space) and it is shared by |
29 | * other drivers/processes, so it could lead to cxl unmapping VMAs |
30 | * from random processes. |
31 | */ |
32 | |
33 | #define CXL_PSEUDO_FS_MAGIC 0x1697697f |
34 | |
35 | static int cxl_fs_cnt; |
36 | static struct vfsmount *cxl_vfs_mount; |
37 | |
38 | static int cxl_fs_init_fs_context(struct fs_context *fc) |
39 | { |
40 | return init_pseudo(fc, CXL_PSEUDO_FS_MAGIC) ? 0 : -ENOMEM; |
41 | } |
42 | |
43 | static struct file_system_type cxl_fs_type = { |
44 | .name = "cxl" , |
45 | .owner = THIS_MODULE, |
46 | .init_fs_context = cxl_fs_init_fs_context, |
47 | .kill_sb = kill_anon_super, |
48 | }; |
49 | |
50 | |
51 | void cxl_release_mapping(struct cxl_context *ctx) |
52 | { |
53 | if (ctx->kernelapi && ctx->mapping) |
54 | simple_release_fs(mount: &cxl_vfs_mount, count: &cxl_fs_cnt); |
55 | } |
56 | |
57 | static struct file *cxl_getfile(const char *name, |
58 | const struct file_operations *fops, |
59 | void *priv, int flags) |
60 | { |
61 | struct file *file; |
62 | struct inode *inode; |
63 | int rc; |
64 | |
65 | /* strongly inspired by anon_inode_getfile() */ |
66 | |
67 | if (fops->owner && !try_module_get(module: fops->owner)) |
68 | return ERR_PTR(error: -ENOENT); |
69 | |
70 | rc = simple_pin_fs(&cxl_fs_type, mount: &cxl_vfs_mount, count: &cxl_fs_cnt); |
71 | if (rc < 0) { |
72 | pr_err("Cannot mount cxl pseudo filesystem: %d\n" , rc); |
73 | file = ERR_PTR(error: rc); |
74 | goto err_module; |
75 | } |
76 | |
77 | inode = alloc_anon_inode(cxl_vfs_mount->mnt_sb); |
78 | if (IS_ERR(ptr: inode)) { |
79 | file = ERR_CAST(ptr: inode); |
80 | goto err_fs; |
81 | } |
82 | |
83 | file = alloc_file_pseudo(inode, cxl_vfs_mount, name, |
84 | flags: flags & (O_ACCMODE | O_NONBLOCK), fops); |
85 | if (IS_ERR(ptr: file)) |
86 | goto err_inode; |
87 | |
88 | file->private_data = priv; |
89 | |
90 | return file; |
91 | |
92 | err_inode: |
93 | iput(inode); |
94 | err_fs: |
95 | simple_release_fs(mount: &cxl_vfs_mount, count: &cxl_fs_cnt); |
96 | err_module: |
97 | module_put(module: fops->owner); |
98 | return file; |
99 | } |
100 | |
101 | struct cxl_context *cxl_dev_context_init(struct pci_dev *dev) |
102 | { |
103 | struct cxl_afu *afu; |
104 | struct cxl_context *ctx; |
105 | int rc; |
106 | |
107 | afu = cxl_pci_to_afu(dev); |
108 | if (IS_ERR(ptr: afu)) |
109 | return ERR_CAST(ptr: afu); |
110 | |
111 | ctx = cxl_context_alloc(); |
112 | if (!ctx) |
113 | return ERR_PTR(error: -ENOMEM); |
114 | |
115 | ctx->kernelapi = true; |
116 | |
117 | /* Make it a slave context. We can promote it later? */ |
118 | rc = cxl_context_init(ctx, afu, master: false); |
119 | if (rc) |
120 | goto err_ctx; |
121 | |
122 | return ctx; |
123 | |
124 | err_ctx: |
125 | kfree(objp: ctx); |
126 | return ERR_PTR(error: rc); |
127 | } |
128 | EXPORT_SYMBOL_GPL(cxl_dev_context_init); |
129 | |
130 | struct cxl_context *cxl_get_context(struct pci_dev *dev) |
131 | { |
132 | return dev->dev.archdata.cxl_ctx; |
133 | } |
134 | EXPORT_SYMBOL_GPL(cxl_get_context); |
135 | |
136 | int cxl_release_context(struct cxl_context *ctx) |
137 | { |
138 | if (ctx->status >= STARTED) |
139 | return -EBUSY; |
140 | |
141 | cxl_context_free(ctx); |
142 | |
143 | return 0; |
144 | } |
145 | EXPORT_SYMBOL_GPL(cxl_release_context); |
146 | |
147 | static irq_hw_number_t cxl_find_afu_irq(struct cxl_context *ctx, int num) |
148 | { |
149 | __u16 range; |
150 | int r; |
151 | |
152 | for (r = 0; r < CXL_IRQ_RANGES; r++) { |
153 | range = ctx->irqs.range[r]; |
154 | if (num < range) { |
155 | return ctx->irqs.offset[r] + num; |
156 | } |
157 | num -= range; |
158 | } |
159 | return 0; |
160 | } |
161 | |
162 | |
163 | int cxl_set_priv(struct cxl_context *ctx, void *priv) |
164 | { |
165 | if (!ctx) |
166 | return -EINVAL; |
167 | |
168 | ctx->priv = priv; |
169 | |
170 | return 0; |
171 | } |
172 | EXPORT_SYMBOL_GPL(cxl_set_priv); |
173 | |
174 | void *cxl_get_priv(struct cxl_context *ctx) |
175 | { |
176 | if (!ctx) |
177 | return ERR_PTR(error: -EINVAL); |
178 | |
179 | return ctx->priv; |
180 | } |
181 | EXPORT_SYMBOL_GPL(cxl_get_priv); |
182 | |
183 | int cxl_allocate_afu_irqs(struct cxl_context *ctx, int num) |
184 | { |
185 | int res; |
186 | irq_hw_number_t hwirq; |
187 | |
188 | if (num == 0) |
189 | num = ctx->afu->pp_irqs; |
190 | res = afu_allocate_irqs(ctx, count: num); |
191 | if (res) |
192 | return res; |
193 | |
194 | if (!cpu_has_feature(CPU_FTR_HVMODE)) { |
195 | /* In a guest, the PSL interrupt is not multiplexed. It was |
196 | * allocated above, and we need to set its handler |
197 | */ |
198 | hwirq = cxl_find_afu_irq(ctx, num: 0); |
199 | if (hwirq) |
200 | cxl_map_irq(adapter: ctx->afu->adapter, hwirq, handler: cxl_ops->psl_interrupt, cookie: ctx, name: "psl" ); |
201 | } |
202 | |
203 | if (ctx->status == STARTED) { |
204 | if (cxl_ops->update_ivtes) |
205 | cxl_ops->update_ivtes(ctx); |
206 | else WARN(1, "BUG: cxl_allocate_afu_irqs must be called prior to starting the context on this platform\n" ); |
207 | } |
208 | |
209 | return res; |
210 | } |
211 | EXPORT_SYMBOL_GPL(cxl_allocate_afu_irqs); |
212 | |
213 | void cxl_free_afu_irqs(struct cxl_context *ctx) |
214 | { |
215 | irq_hw_number_t hwirq; |
216 | unsigned int virq; |
217 | |
218 | if (!cpu_has_feature(CPU_FTR_HVMODE)) { |
219 | hwirq = cxl_find_afu_irq(ctx, num: 0); |
220 | if (hwirq) { |
221 | virq = irq_find_mapping(NULL, hwirq); |
222 | if (virq) |
223 | cxl_unmap_irq(virq, cookie: ctx); |
224 | } |
225 | } |
226 | afu_irq_name_free(ctx); |
227 | cxl_ops->release_irq_ranges(&ctx->irqs, ctx->afu->adapter); |
228 | } |
229 | EXPORT_SYMBOL_GPL(cxl_free_afu_irqs); |
230 | |
231 | int cxl_map_afu_irq(struct cxl_context *ctx, int num, |
232 | irq_handler_t handler, void *cookie, char *name) |
233 | { |
234 | irq_hw_number_t hwirq; |
235 | |
236 | /* |
237 | * Find interrupt we are to register. |
238 | */ |
239 | hwirq = cxl_find_afu_irq(ctx, num); |
240 | if (!hwirq) |
241 | return -ENOENT; |
242 | |
243 | return cxl_map_irq(adapter: ctx->afu->adapter, hwirq, handler, cookie, name); |
244 | } |
245 | EXPORT_SYMBOL_GPL(cxl_map_afu_irq); |
246 | |
247 | void cxl_unmap_afu_irq(struct cxl_context *ctx, int num, void *cookie) |
248 | { |
249 | irq_hw_number_t hwirq; |
250 | unsigned int virq; |
251 | |
252 | hwirq = cxl_find_afu_irq(ctx, num); |
253 | if (!hwirq) |
254 | return; |
255 | |
256 | virq = irq_find_mapping(NULL, hwirq); |
257 | if (virq) |
258 | cxl_unmap_irq(virq, cookie); |
259 | } |
260 | EXPORT_SYMBOL_GPL(cxl_unmap_afu_irq); |
261 | |
262 | /* |
263 | * Start a context |
264 | * Code here similar to afu_ioctl_start_work(). |
265 | */ |
266 | int cxl_start_context(struct cxl_context *ctx, u64 wed, |
267 | struct task_struct *task) |
268 | { |
269 | int rc = 0; |
270 | bool kernel = true; |
271 | |
272 | pr_devel("%s: pe: %i\n" , __func__, ctx->pe); |
273 | |
274 | mutex_lock(&ctx->status_mutex); |
275 | if (ctx->status == STARTED) |
276 | goto out; /* already started */ |
277 | |
278 | /* |
279 | * Increment the mapped context count for adapter. This also checks |
280 | * if adapter_context_lock is taken. |
281 | */ |
282 | rc = cxl_adapter_context_get(adapter: ctx->afu->adapter); |
283 | if (rc) |
284 | goto out; |
285 | |
286 | if (task) { |
287 | ctx->pid = get_task_pid(task, type: PIDTYPE_PID); |
288 | kernel = false; |
289 | |
290 | /* acquire a reference to the task's mm */ |
291 | ctx->mm = get_task_mm(current); |
292 | |
293 | /* ensure this mm_struct can't be freed */ |
294 | cxl_context_mm_count_get(ctx); |
295 | |
296 | if (ctx->mm) { |
297 | /* decrement the use count from above */ |
298 | mmput(ctx->mm); |
299 | /* make TLBIs for this context global */ |
300 | mm_context_add_copro(ctx->mm); |
301 | } |
302 | } |
303 | |
304 | /* |
305 | * Increment driver use count. Enables global TLBIs for hash |
306 | * and callbacks to handle the segment table |
307 | */ |
308 | cxl_ctx_get(); |
309 | |
310 | /* See the comment in afu_ioctl_start_work() */ |
311 | smp_mb(); |
312 | |
313 | if ((rc = cxl_ops->attach_process(ctx, kernel, wed, 0))) { |
314 | put_pid(pid: ctx->pid); |
315 | ctx->pid = NULL; |
316 | cxl_adapter_context_put(adapter: ctx->afu->adapter); |
317 | cxl_ctx_put(); |
318 | if (task) { |
319 | cxl_context_mm_count_put(ctx); |
320 | if (ctx->mm) |
321 | mm_context_remove_copro(ctx->mm); |
322 | } |
323 | goto out; |
324 | } |
325 | |
326 | ctx->status = STARTED; |
327 | out: |
328 | mutex_unlock(lock: &ctx->status_mutex); |
329 | return rc; |
330 | } |
331 | EXPORT_SYMBOL_GPL(cxl_start_context); |
332 | |
333 | int cxl_process_element(struct cxl_context *ctx) |
334 | { |
335 | return ctx->external_pe; |
336 | } |
337 | EXPORT_SYMBOL_GPL(cxl_process_element); |
338 | |
339 | /* Stop a context. Returns 0 on success, otherwise -Errno */ |
340 | int cxl_stop_context(struct cxl_context *ctx) |
341 | { |
342 | return __detach_context(ctx); |
343 | } |
344 | EXPORT_SYMBOL_GPL(cxl_stop_context); |
345 | |
346 | void cxl_set_master(struct cxl_context *ctx) |
347 | { |
348 | ctx->master = true; |
349 | } |
350 | EXPORT_SYMBOL_GPL(cxl_set_master); |
351 | |
352 | /* wrappers around afu_* file ops which are EXPORTED */ |
353 | int cxl_fd_open(struct inode *inode, struct file *file) |
354 | { |
355 | return afu_open(inode, file); |
356 | } |
357 | EXPORT_SYMBOL_GPL(cxl_fd_open); |
358 | int cxl_fd_release(struct inode *inode, struct file *file) |
359 | { |
360 | return afu_release(inode, file); |
361 | } |
362 | EXPORT_SYMBOL_GPL(cxl_fd_release); |
363 | long cxl_fd_ioctl(struct file *file, unsigned int cmd, unsigned long arg) |
364 | { |
365 | return afu_ioctl(file, cmd, arg); |
366 | } |
367 | EXPORT_SYMBOL_GPL(cxl_fd_ioctl); |
368 | int cxl_fd_mmap(struct file *file, struct vm_area_struct *vm) |
369 | { |
370 | return afu_mmap(file, vm); |
371 | } |
372 | EXPORT_SYMBOL_GPL(cxl_fd_mmap); |
373 | __poll_t cxl_fd_poll(struct file *file, struct poll_table_struct *poll) |
374 | { |
375 | return afu_poll(file, poll); |
376 | } |
377 | EXPORT_SYMBOL_GPL(cxl_fd_poll); |
378 | ssize_t cxl_fd_read(struct file *file, char __user *buf, size_t count, |
379 | loff_t *off) |
380 | { |
381 | return afu_read(file, buf, count, off); |
382 | } |
383 | EXPORT_SYMBOL_GPL(cxl_fd_read); |
384 | |
385 | #define PATCH_FOPS(NAME) if (!fops->NAME) fops->NAME = afu_fops.NAME |
386 | |
387 | /* Get a struct file and fd for a context and attach the ops */ |
388 | struct file *cxl_get_fd(struct cxl_context *ctx, struct file_operations *fops, |
389 | int *fd) |
390 | { |
391 | struct file *file; |
392 | int rc, flags, fdtmp; |
393 | char *name = NULL; |
394 | |
395 | /* only allow one per context */ |
396 | if (ctx->mapping) |
397 | return ERR_PTR(error: -EEXIST); |
398 | |
399 | flags = O_RDWR | O_CLOEXEC; |
400 | |
401 | /* This code is similar to anon_inode_getfd() */ |
402 | rc = get_unused_fd_flags(flags); |
403 | if (rc < 0) |
404 | return ERR_PTR(error: rc); |
405 | fdtmp = rc; |
406 | |
407 | /* |
408 | * Patch the file ops. Needs to be careful that this is rentrant safe. |
409 | */ |
410 | if (fops) { |
411 | PATCH_FOPS(open); |
412 | PATCH_FOPS(poll); |
413 | PATCH_FOPS(read); |
414 | PATCH_FOPS(release); |
415 | PATCH_FOPS(unlocked_ioctl); |
416 | PATCH_FOPS(compat_ioctl); |
417 | PATCH_FOPS(mmap); |
418 | } else /* use default ops */ |
419 | fops = (struct file_operations *)&afu_fops; |
420 | |
421 | name = kasprintf(GFP_KERNEL, fmt: "cxl:%d" , ctx->pe); |
422 | file = cxl_getfile(name, fops, priv: ctx, flags); |
423 | kfree(objp: name); |
424 | if (IS_ERR(ptr: file)) |
425 | goto err_fd; |
426 | |
427 | cxl_context_set_mapping(ctx, mapping: file->f_mapping); |
428 | *fd = fdtmp; |
429 | return file; |
430 | |
431 | err_fd: |
432 | put_unused_fd(fd: fdtmp); |
433 | return NULL; |
434 | } |
435 | EXPORT_SYMBOL_GPL(cxl_get_fd); |
436 | |
437 | struct cxl_context *cxl_fops_get_context(struct file *file) |
438 | { |
439 | return file->private_data; |
440 | } |
441 | EXPORT_SYMBOL_GPL(cxl_fops_get_context); |
442 | |
443 | void cxl_set_driver_ops(struct cxl_context *ctx, |
444 | struct cxl_afu_driver_ops *ops) |
445 | { |
446 | WARN_ON(!ops->fetch_event || !ops->event_delivered); |
447 | atomic_set(v: &ctx->afu_driver_events, i: 0); |
448 | ctx->afu_driver_ops = ops; |
449 | } |
450 | EXPORT_SYMBOL_GPL(cxl_set_driver_ops); |
451 | |
452 | void cxl_context_events_pending(struct cxl_context *ctx, |
453 | unsigned int new_events) |
454 | { |
455 | atomic_add(i: new_events, v: &ctx->afu_driver_events); |
456 | wake_up_all(&ctx->wq); |
457 | } |
458 | EXPORT_SYMBOL_GPL(cxl_context_events_pending); |
459 | |
460 | int cxl_start_work(struct cxl_context *ctx, |
461 | struct cxl_ioctl_start_work *work) |
462 | { |
463 | int rc; |
464 | |
465 | /* code taken from afu_ioctl_start_work */ |
466 | if (!(work->flags & CXL_START_WORK_NUM_IRQS)) |
467 | work->num_interrupts = ctx->afu->pp_irqs; |
468 | else if ((work->num_interrupts < ctx->afu->pp_irqs) || |
469 | (work->num_interrupts > ctx->afu->irqs_max)) { |
470 | return -EINVAL; |
471 | } |
472 | |
473 | rc = afu_register_irqs(ctx, count: work->num_interrupts); |
474 | if (rc) |
475 | return rc; |
476 | |
477 | rc = cxl_start_context(ctx, work->work_element_descriptor, current); |
478 | if (rc < 0) { |
479 | afu_release_irqs(ctx, cookie: ctx); |
480 | return rc; |
481 | } |
482 | |
483 | return 0; |
484 | } |
485 | EXPORT_SYMBOL_GPL(cxl_start_work); |
486 | |
487 | void __iomem *cxl_psa_map(struct cxl_context *ctx) |
488 | { |
489 | if (ctx->status != STARTED) |
490 | return NULL; |
491 | |
492 | pr_devel("%s: psn_phys%llx size:%llx\n" , |
493 | __func__, ctx->psn_phys, ctx->psn_size); |
494 | return ioremap(offset: ctx->psn_phys, size: ctx->psn_size); |
495 | } |
496 | EXPORT_SYMBOL_GPL(cxl_psa_map); |
497 | |
498 | void cxl_psa_unmap(void __iomem *addr) |
499 | { |
500 | iounmap(addr); |
501 | } |
502 | EXPORT_SYMBOL_GPL(cxl_psa_unmap); |
503 | |
504 | int cxl_afu_reset(struct cxl_context *ctx) |
505 | { |
506 | struct cxl_afu *afu = ctx->afu; |
507 | int rc; |
508 | |
509 | rc = cxl_ops->afu_reset(afu); |
510 | if (rc) |
511 | return rc; |
512 | |
513 | return cxl_ops->afu_check_and_enable(afu); |
514 | } |
515 | EXPORT_SYMBOL_GPL(cxl_afu_reset); |
516 | |
517 | void cxl_perst_reloads_same_image(struct cxl_afu *afu, |
518 | bool perst_reloads_same_image) |
519 | { |
520 | afu->adapter->perst_same_image = perst_reloads_same_image; |
521 | } |
522 | EXPORT_SYMBOL_GPL(cxl_perst_reloads_same_image); |
523 | |
524 | ssize_t cxl_read_adapter_vpd(struct pci_dev *dev, void *buf, size_t count) |
525 | { |
526 | struct cxl_afu *afu = cxl_pci_to_afu(dev); |
527 | if (IS_ERR(ptr: afu)) |
528 | return -ENODEV; |
529 | |
530 | return cxl_ops->read_adapter_vpd(afu->adapter, buf, count); |
531 | } |
532 | EXPORT_SYMBOL_GPL(cxl_read_adapter_vpd); |
533 | |