1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Driver for FPGA Accelerated Function Unit (AFU)
4 *
5 * Copyright (C) 2017-2018 Intel Corporation, Inc.
6 *
7 * Authors:
8 * Wu Hao <hao.wu@intel.com>
9 * Xiao Guangrong <guangrong.xiao@linux.intel.com>
10 * Joseph Grecco <joe.grecco@intel.com>
11 * Enno Luebbers <enno.luebbers@intel.com>
12 * Tim Whisonant <tim.whisonant@intel.com>
13 * Ananda Ravuri <ananda.ravuri@intel.com>
14 * Henry Mitchel <henry.mitchel@intel.com>
15 */
16
17#include <linux/kernel.h>
18#include <linux/module.h>
19#include <linux/uaccess.h>
20#include <linux/fpga-dfl.h>
21
22#include "dfl-afu.h"
23
24/**
25 * port_enable - enable a port
26 * @pdev: port platform device.
27 *
28 * Enable Port by clear the port soft reset bit, which is set by default.
29 * The AFU is unable to respond to any MMIO access while in reset.
30 * port_enable function should only be used after port_disable function.
31 */
32static void port_enable(struct platform_device *pdev)
33{
34 struct dfl_feature_platform_data *pdata = dev_get_platdata(&pdev->dev);
35 void __iomem *base;
36 u64 v;
37
38 WARN_ON(!pdata->disable_count);
39
40 if (--pdata->disable_count != 0)
41 return;
42
43 base = dfl_get_feature_ioaddr_by_id(&pdev->dev, PORT_FEATURE_ID_HEADER);
44
45 /* Clear port soft reset */
46 v = readq(base + PORT_HDR_CTRL);
47 v &= ~PORT_CTRL_SFTRST;
48 writeq(v, base + PORT_HDR_CTRL);
49}
50
51#define RST_POLL_INVL 10 /* us */
52#define RST_POLL_TIMEOUT 1000 /* us */
53
54/**
55 * port_disable - disable a port
56 * @pdev: port platform device.
57 *
58 * Disable Port by setting the port soft reset bit, it puts the port into
59 * reset.
60 */
61static int port_disable(struct platform_device *pdev)
62{
63 struct dfl_feature_platform_data *pdata = dev_get_platdata(&pdev->dev);
64 void __iomem *base;
65 u64 v;
66
67 if (pdata->disable_count++ != 0)
68 return 0;
69
70 base = dfl_get_feature_ioaddr_by_id(&pdev->dev, PORT_FEATURE_ID_HEADER);
71
72 /* Set port soft reset */
73 v = readq(base + PORT_HDR_CTRL);
74 v |= PORT_CTRL_SFTRST;
75 writeq(v, base + PORT_HDR_CTRL);
76
77 /*
78 * HW sets ack bit to 1 when all outstanding requests have been drained
79 * on this port and minimum soft reset pulse width has elapsed.
80 * Driver polls port_soft_reset_ack to determine if reset done by HW.
81 */
82 if (readq_poll_timeout(base + PORT_HDR_CTRL, v, v & PORT_CTRL_SFTRST,
83 RST_POLL_INVL, RST_POLL_TIMEOUT)) {
84 dev_err(&pdev->dev, "timeout, fail to reset device\n");
85 return -ETIMEDOUT;
86 }
87
88 return 0;
89}
90
91/*
92 * This function resets the FPGA Port and its accelerator (AFU) by function
93 * __port_disable and __port_enable (set port soft reset bit and then clear
94 * it). Userspace can do Port reset at any time, e.g. during DMA or Partial
95 * Reconfiguration. But it should never cause any system level issue, only
96 * functional failure (e.g. DMA or PR operation failure) and be recoverable
97 * from the failure.
98 *
99 * Note: the accelerator (AFU) is not accessible when its port is in reset
100 * (disabled). Any attempts on MMIO access to AFU while in reset, will
101 * result errors reported via port error reporting sub feature (if present).
102 */
103static int __port_reset(struct platform_device *pdev)
104{
105 int ret;
106
107 ret = port_disable(pdev);
108 if (!ret)
109 port_enable(pdev);
110
111 return ret;
112}
113
114static int port_reset(struct platform_device *pdev)
115{
116 struct dfl_feature_platform_data *pdata = dev_get_platdata(&pdev->dev);
117 int ret;
118
119 mutex_lock(&pdata->lock);
120 ret = __port_reset(pdev);
121 mutex_unlock(&pdata->lock);
122
123 return ret;
124}
125
126static int port_get_id(struct platform_device *pdev)
127{
128 void __iomem *base;
129
130 base = dfl_get_feature_ioaddr_by_id(&pdev->dev, PORT_FEATURE_ID_HEADER);
131
132 return FIELD_GET(PORT_CAP_PORT_NUM, readq(base + PORT_HDR_CAP));
133}
134
135static ssize_t
136id_show(struct device *dev, struct device_attribute *attr, char *buf)
137{
138 int id = port_get_id(to_platform_device(dev));
139
140 return scnprintf(buf, PAGE_SIZE, "%d\n", id);
141}
142static DEVICE_ATTR_RO(id);
143
144static const struct attribute *port_hdr_attrs[] = {
145 &dev_attr_id.attr,
146 NULL,
147};
148
149static int port_hdr_init(struct platform_device *pdev,
150 struct dfl_feature *feature)
151{
152 dev_dbg(&pdev->dev, "PORT HDR Init.\n");
153
154 port_reset(pdev);
155
156 return sysfs_create_files(&pdev->dev.kobj, port_hdr_attrs);
157}
158
159static void port_hdr_uinit(struct platform_device *pdev,
160 struct dfl_feature *feature)
161{
162 dev_dbg(&pdev->dev, "PORT HDR UInit.\n");
163
164 sysfs_remove_files(&pdev->dev.kobj, port_hdr_attrs);
165}
166
167static long
168port_hdr_ioctl(struct platform_device *pdev, struct dfl_feature *feature,
169 unsigned int cmd, unsigned long arg)
170{
171 long ret;
172
173 switch (cmd) {
174 case DFL_FPGA_PORT_RESET:
175 if (!arg)
176 ret = port_reset(pdev);
177 else
178 ret = -EINVAL;
179 break;
180 default:
181 dev_dbg(&pdev->dev, "%x cmd not handled", cmd);
182 ret = -ENODEV;
183 }
184
185 return ret;
186}
187
188static const struct dfl_feature_ops port_hdr_ops = {
189 .init = port_hdr_init,
190 .uinit = port_hdr_uinit,
191 .ioctl = port_hdr_ioctl,
192};
193
194static ssize_t
195afu_id_show(struct device *dev, struct device_attribute *attr, char *buf)
196{
197 struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
198 void __iomem *base;
199 u64 guidl, guidh;
200
201 base = dfl_get_feature_ioaddr_by_id(dev, PORT_FEATURE_ID_AFU);
202
203 mutex_lock(&pdata->lock);
204 if (pdata->disable_count) {
205 mutex_unlock(&pdata->lock);
206 return -EBUSY;
207 }
208
209 guidl = readq(base + GUID_L);
210 guidh = readq(base + GUID_H);
211 mutex_unlock(&pdata->lock);
212
213 return scnprintf(buf, PAGE_SIZE, "%016llx%016llx\n", guidh, guidl);
214}
215static DEVICE_ATTR_RO(afu_id);
216
217static const struct attribute *port_afu_attrs[] = {
218 &dev_attr_afu_id.attr,
219 NULL
220};
221
222static int port_afu_init(struct platform_device *pdev,
223 struct dfl_feature *feature)
224{
225 struct resource *res = &pdev->resource[feature->resource_index];
226 int ret;
227
228 dev_dbg(&pdev->dev, "PORT AFU Init.\n");
229
230 ret = afu_mmio_region_add(dev_get_platdata(&pdev->dev),
231 DFL_PORT_REGION_INDEX_AFU, resource_size(res),
232 res->start, DFL_PORT_REGION_READ |
233 DFL_PORT_REGION_WRITE | DFL_PORT_REGION_MMAP);
234 if (ret)
235 return ret;
236
237 return sysfs_create_files(&pdev->dev.kobj, port_afu_attrs);
238}
239
240static void port_afu_uinit(struct platform_device *pdev,
241 struct dfl_feature *feature)
242{
243 dev_dbg(&pdev->dev, "PORT AFU UInit.\n");
244
245 sysfs_remove_files(&pdev->dev.kobj, port_afu_attrs);
246}
247
248static const struct dfl_feature_ops port_afu_ops = {
249 .init = port_afu_init,
250 .uinit = port_afu_uinit,
251};
252
253static struct dfl_feature_driver port_feature_drvs[] = {
254 {
255 .id = PORT_FEATURE_ID_HEADER,
256 .ops = &port_hdr_ops,
257 },
258 {
259 .id = PORT_FEATURE_ID_AFU,
260 .ops = &port_afu_ops,
261 },
262 {
263 .ops = NULL,
264 }
265};
266
267static int afu_open(struct inode *inode, struct file *filp)
268{
269 struct platform_device *fdev = dfl_fpga_inode_to_feature_dev(inode);
270 struct dfl_feature_platform_data *pdata;
271 int ret;
272
273 pdata = dev_get_platdata(&fdev->dev);
274 if (WARN_ON(!pdata))
275 return -ENODEV;
276
277 ret = dfl_feature_dev_use_begin(pdata);
278 if (ret)
279 return ret;
280
281 dev_dbg(&fdev->dev, "Device File Open\n");
282 filp->private_data = fdev;
283
284 return 0;
285}
286
287static int afu_release(struct inode *inode, struct file *filp)
288{
289 struct platform_device *pdev = filp->private_data;
290 struct dfl_feature_platform_data *pdata;
291
292 dev_dbg(&pdev->dev, "Device File Release\n");
293
294 pdata = dev_get_platdata(&pdev->dev);
295
296 mutex_lock(&pdata->lock);
297 __port_reset(pdev);
298 afu_dma_region_destroy(pdata);
299 mutex_unlock(&pdata->lock);
300
301 dfl_feature_dev_use_end(pdata);
302
303 return 0;
304}
305
306static long afu_ioctl_check_extension(struct dfl_feature_platform_data *pdata,
307 unsigned long arg)
308{
309 /* No extension support for now */
310 return 0;
311}
312
313static long
314afu_ioctl_get_info(struct dfl_feature_platform_data *pdata, void __user *arg)
315{
316 struct dfl_fpga_port_info info;
317 struct dfl_afu *afu;
318 unsigned long minsz;
319
320 minsz = offsetofend(struct dfl_fpga_port_info, num_umsgs);
321
322 if (copy_from_user(&info, arg, minsz))
323 return -EFAULT;
324
325 if (info.argsz < minsz)
326 return -EINVAL;
327
328 mutex_lock(&pdata->lock);
329 afu = dfl_fpga_pdata_get_private(pdata);
330 info.flags = 0;
331 info.num_regions = afu->num_regions;
332 info.num_umsgs = afu->num_umsgs;
333 mutex_unlock(&pdata->lock);
334
335 if (copy_to_user(arg, &info, sizeof(info)))
336 return -EFAULT;
337
338 return 0;
339}
340
341static long afu_ioctl_get_region_info(struct dfl_feature_platform_data *pdata,
342 void __user *arg)
343{
344 struct dfl_fpga_port_region_info rinfo;
345 struct dfl_afu_mmio_region region;
346 unsigned long minsz;
347 long ret;
348
349 minsz = offsetofend(struct dfl_fpga_port_region_info, offset);
350
351 if (copy_from_user(&rinfo, arg, minsz))
352 return -EFAULT;
353
354 if (rinfo.argsz < minsz || rinfo.padding)
355 return -EINVAL;
356
357 ret = afu_mmio_region_get_by_index(pdata, rinfo.index, &region);
358 if (ret)
359 return ret;
360
361 rinfo.flags = region.flags;
362 rinfo.size = region.size;
363 rinfo.offset = region.offset;
364
365 if (copy_to_user(arg, &rinfo, sizeof(rinfo)))
366 return -EFAULT;
367
368 return 0;
369}
370
371static long
372afu_ioctl_dma_map(struct dfl_feature_platform_data *pdata, void __user *arg)
373{
374 struct dfl_fpga_port_dma_map map;
375 unsigned long minsz;
376 long ret;
377
378 minsz = offsetofend(struct dfl_fpga_port_dma_map, iova);
379
380 if (copy_from_user(&map, arg, minsz))
381 return -EFAULT;
382
383 if (map.argsz < minsz || map.flags)
384 return -EINVAL;
385
386 ret = afu_dma_map_region(pdata, map.user_addr, map.length, &map.iova);
387 if (ret)
388 return ret;
389
390 if (copy_to_user(arg, &map, sizeof(map))) {
391 afu_dma_unmap_region(pdata, map.iova);
392 return -EFAULT;
393 }
394
395 dev_dbg(&pdata->dev->dev, "dma map: ua=%llx, len=%llx, iova=%llx\n",
396 (unsigned long long)map.user_addr,
397 (unsigned long long)map.length,
398 (unsigned long long)map.iova);
399
400 return 0;
401}
402
403static long
404afu_ioctl_dma_unmap(struct dfl_feature_platform_data *pdata, void __user *arg)
405{
406 struct dfl_fpga_port_dma_unmap unmap;
407 unsigned long minsz;
408
409 minsz = offsetofend(struct dfl_fpga_port_dma_unmap, iova);
410
411 if (copy_from_user(&unmap, arg, minsz))
412 return -EFAULT;
413
414 if (unmap.argsz < minsz || unmap.flags)
415 return -EINVAL;
416
417 return afu_dma_unmap_region(pdata, unmap.iova);
418}
419
420static long afu_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
421{
422 struct platform_device *pdev = filp->private_data;
423 struct dfl_feature_platform_data *pdata;
424 struct dfl_feature *f;
425 long ret;
426
427 dev_dbg(&pdev->dev, "%s cmd 0x%x\n", __func__, cmd);
428
429 pdata = dev_get_platdata(&pdev->dev);
430
431 switch (cmd) {
432 case DFL_FPGA_GET_API_VERSION:
433 return DFL_FPGA_API_VERSION;
434 case DFL_FPGA_CHECK_EXTENSION:
435 return afu_ioctl_check_extension(pdata, arg);
436 case DFL_FPGA_PORT_GET_INFO:
437 return afu_ioctl_get_info(pdata, (void __user *)arg);
438 case DFL_FPGA_PORT_GET_REGION_INFO:
439 return afu_ioctl_get_region_info(pdata, (void __user *)arg);
440 case DFL_FPGA_PORT_DMA_MAP:
441 return afu_ioctl_dma_map(pdata, (void __user *)arg);
442 case DFL_FPGA_PORT_DMA_UNMAP:
443 return afu_ioctl_dma_unmap(pdata, (void __user *)arg);
444 default:
445 /*
446 * Let sub-feature's ioctl function to handle the cmd
447 * Sub-feature's ioctl returns -ENODEV when cmd is not
448 * handled in this sub feature, and returns 0 and other
449 * error code if cmd is handled.
450 */
451 dfl_fpga_dev_for_each_feature(pdata, f)
452 if (f->ops && f->ops->ioctl) {
453 ret = f->ops->ioctl(pdev, f, cmd, arg);
454 if (ret != -ENODEV)
455 return ret;
456 }
457 }
458
459 return -EINVAL;
460}
461
462static int afu_mmap(struct file *filp, struct vm_area_struct *vma)
463{
464 struct platform_device *pdev = filp->private_data;
465 struct dfl_feature_platform_data *pdata;
466 u64 size = vma->vm_end - vma->vm_start;
467 struct dfl_afu_mmio_region region;
468 u64 offset;
469 int ret;
470
471 if (!(vma->vm_flags & VM_SHARED))
472 return -EINVAL;
473
474 pdata = dev_get_platdata(&pdev->dev);
475
476 offset = vma->vm_pgoff << PAGE_SHIFT;
477 ret = afu_mmio_region_get_by_offset(pdata, offset, size, &region);
478 if (ret)
479 return ret;
480
481 if (!(region.flags & DFL_PORT_REGION_MMAP))
482 return -EINVAL;
483
484 if ((vma->vm_flags & VM_READ) && !(region.flags & DFL_PORT_REGION_READ))
485 return -EPERM;
486
487 if ((vma->vm_flags & VM_WRITE) &&
488 !(region.flags & DFL_PORT_REGION_WRITE))
489 return -EPERM;
490
491 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
492
493 return remap_pfn_range(vma, vma->vm_start,
494 (region.phys + (offset - region.offset)) >> PAGE_SHIFT,
495 size, vma->vm_page_prot);
496}
497
498static const struct file_operations afu_fops = {
499 .owner = THIS_MODULE,
500 .open = afu_open,
501 .release = afu_release,
502 .unlocked_ioctl = afu_ioctl,
503 .mmap = afu_mmap,
504};
505
506static int afu_dev_init(struct platform_device *pdev)
507{
508 struct dfl_feature_platform_data *pdata = dev_get_platdata(&pdev->dev);
509 struct dfl_afu *afu;
510
511 afu = devm_kzalloc(&pdev->dev, sizeof(*afu), GFP_KERNEL);
512 if (!afu)
513 return -ENOMEM;
514
515 afu->pdata = pdata;
516
517 mutex_lock(&pdata->lock);
518 dfl_fpga_pdata_set_private(pdata, afu);
519 afu_mmio_region_init(pdata);
520 afu_dma_region_init(pdata);
521 mutex_unlock(&pdata->lock);
522
523 return 0;
524}
525
526static int afu_dev_destroy(struct platform_device *pdev)
527{
528 struct dfl_feature_platform_data *pdata = dev_get_platdata(&pdev->dev);
529 struct dfl_afu *afu;
530
531 mutex_lock(&pdata->lock);
532 afu = dfl_fpga_pdata_get_private(pdata);
533 afu_mmio_region_destroy(pdata);
534 afu_dma_region_destroy(pdata);
535 dfl_fpga_pdata_set_private(pdata, NULL);
536 mutex_unlock(&pdata->lock);
537
538 return 0;
539}
540
541static int port_enable_set(struct platform_device *pdev, bool enable)
542{
543 struct dfl_feature_platform_data *pdata = dev_get_platdata(&pdev->dev);
544 int ret = 0;
545
546 mutex_lock(&pdata->lock);
547 if (enable)
548 port_enable(pdev);
549 else
550 ret = port_disable(pdev);
551 mutex_unlock(&pdata->lock);
552
553 return ret;
554}
555
556static struct dfl_fpga_port_ops afu_port_ops = {
557 .name = DFL_FPGA_FEATURE_DEV_PORT,
558 .owner = THIS_MODULE,
559 .get_id = port_get_id,
560 .enable_set = port_enable_set,
561};
562
563static int afu_probe(struct platform_device *pdev)
564{
565 int ret;
566
567 dev_dbg(&pdev->dev, "%s\n", __func__);
568
569 ret = afu_dev_init(pdev);
570 if (ret)
571 goto exit;
572
573 ret = dfl_fpga_dev_feature_init(pdev, port_feature_drvs);
574 if (ret)
575 goto dev_destroy;
576
577 ret = dfl_fpga_dev_ops_register(pdev, &afu_fops, THIS_MODULE);
578 if (ret) {
579 dfl_fpga_dev_feature_uinit(pdev);
580 goto dev_destroy;
581 }
582
583 return 0;
584
585dev_destroy:
586 afu_dev_destroy(pdev);
587exit:
588 return ret;
589}
590
591static int afu_remove(struct platform_device *pdev)
592{
593 dev_dbg(&pdev->dev, "%s\n", __func__);
594
595 dfl_fpga_dev_ops_unregister(pdev);
596 dfl_fpga_dev_feature_uinit(pdev);
597 afu_dev_destroy(pdev);
598
599 return 0;
600}
601
602static struct platform_driver afu_driver = {
603 .driver = {
604 .name = DFL_FPGA_FEATURE_DEV_PORT,
605 },
606 .probe = afu_probe,
607 .remove = afu_remove,
608};
609
610static int __init afu_init(void)
611{
612 int ret;
613
614 dfl_fpga_port_ops_add(&afu_port_ops);
615
616 ret = platform_driver_register(&afu_driver);
617 if (ret)
618 dfl_fpga_port_ops_del(&afu_port_ops);
619
620 return ret;
621}
622
623static void __exit afu_exit(void)
624{
625 platform_driver_unregister(&afu_driver);
626
627 dfl_fpga_port_ops_del(&afu_port_ops);
628}
629
630module_init(afu_init);
631module_exit(afu_exit);
632
633MODULE_DESCRIPTION("FPGA Accelerated Function Unit driver");
634MODULE_AUTHOR("Intel Corporation");
635MODULE_LICENSE("GPL v2");
636MODULE_ALIAS("platform:dfl-port");
637