1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Host side test driver to test endpoint functionality |
4 | * |
5 | * Copyright (C) 2017 Texas Instruments |
6 | * Author: Kishon Vijay Abraham I <kishon@ti.com> |
7 | */ |
8 | |
9 | #include <linux/crc32.h> |
10 | #include <linux/delay.h> |
11 | #include <linux/fs.h> |
12 | #include <linux/io.h> |
13 | #include <linux/interrupt.h> |
14 | #include <linux/irq.h> |
15 | #include <linux/miscdevice.h> |
16 | #include <linux/module.h> |
17 | #include <linux/mutex.h> |
18 | #include <linux/random.h> |
19 | #include <linux/slab.h> |
20 | #include <linux/uaccess.h> |
21 | #include <linux/pci.h> |
22 | #include <linux/pci_ids.h> |
23 | |
24 | #include <linux/pci_regs.h> |
25 | |
26 | #include <uapi/linux/pcitest.h> |
27 | |
28 | #define DRV_MODULE_NAME "pci-endpoint-test" |
29 | |
30 | #define IRQ_TYPE_UNDEFINED -1 |
31 | #define IRQ_TYPE_LEGACY 0 |
32 | #define IRQ_TYPE_MSI 1 |
33 | #define IRQ_TYPE_MSIX 2 |
34 | |
35 | #define PCI_ENDPOINT_TEST_MAGIC 0x0 |
36 | |
37 | #define PCI_ENDPOINT_TEST_COMMAND 0x4 |
38 | #define COMMAND_RAISE_LEGACY_IRQ BIT(0) |
39 | #define COMMAND_RAISE_MSI_IRQ BIT(1) |
40 | #define COMMAND_RAISE_MSIX_IRQ BIT(2) |
41 | #define COMMAND_READ BIT(3) |
42 | #define COMMAND_WRITE BIT(4) |
43 | #define COMMAND_COPY BIT(5) |
44 | |
45 | #define PCI_ENDPOINT_TEST_STATUS 0x8 |
46 | #define STATUS_READ_SUCCESS BIT(0) |
47 | #define STATUS_READ_FAIL BIT(1) |
48 | #define STATUS_WRITE_SUCCESS BIT(2) |
49 | #define STATUS_WRITE_FAIL BIT(3) |
50 | #define STATUS_COPY_SUCCESS BIT(4) |
51 | #define STATUS_COPY_FAIL BIT(5) |
52 | #define STATUS_IRQ_RAISED BIT(6) |
53 | #define STATUS_SRC_ADDR_INVALID BIT(7) |
54 | #define STATUS_DST_ADDR_INVALID BIT(8) |
55 | |
56 | #define PCI_ENDPOINT_TEST_LOWER_SRC_ADDR 0x0c |
57 | #define PCI_ENDPOINT_TEST_UPPER_SRC_ADDR 0x10 |
58 | |
59 | #define PCI_ENDPOINT_TEST_LOWER_DST_ADDR 0x14 |
60 | #define PCI_ENDPOINT_TEST_UPPER_DST_ADDR 0x18 |
61 | |
62 | #define PCI_ENDPOINT_TEST_SIZE 0x1c |
63 | #define PCI_ENDPOINT_TEST_CHECKSUM 0x20 |
64 | |
65 | #define PCI_ENDPOINT_TEST_IRQ_TYPE 0x24 |
66 | #define PCI_ENDPOINT_TEST_IRQ_NUMBER 0x28 |
67 | |
68 | #define PCI_ENDPOINT_TEST_FLAGS 0x2c |
69 | #define FLAG_USE_DMA BIT(0) |
70 | |
71 | #define PCI_DEVICE_ID_TI_AM654 0xb00c |
72 | #define PCI_DEVICE_ID_TI_J7200 0xb00f |
73 | #define PCI_DEVICE_ID_TI_AM64 0xb010 |
74 | #define PCI_DEVICE_ID_TI_J721S2 0xb013 |
75 | #define PCI_DEVICE_ID_LS1088A 0x80c0 |
76 | #define PCI_DEVICE_ID_IMX8 0x0808 |
77 | |
78 | #define is_am654_pci_dev(pdev) \ |
79 | ((pdev)->device == PCI_DEVICE_ID_TI_AM654) |
80 | |
81 | #define PCI_DEVICE_ID_RENESAS_R8A774A1 0x0028 |
82 | #define PCI_DEVICE_ID_RENESAS_R8A774B1 0x002b |
83 | #define PCI_DEVICE_ID_RENESAS_R8A774C0 0x002d |
84 | #define PCI_DEVICE_ID_RENESAS_R8A774E1 0x0025 |
85 | #define PCI_DEVICE_ID_RENESAS_R8A779F0 0x0031 |
86 | |
87 | static DEFINE_IDA(pci_endpoint_test_ida); |
88 | |
89 | #define to_endpoint_test(priv) container_of((priv), struct pci_endpoint_test, \ |
90 | miscdev) |
91 | |
92 | static bool no_msi; |
93 | module_param(no_msi, bool, 0444); |
94 | MODULE_PARM_DESC(no_msi, "Disable MSI interrupt in pci_endpoint_test" ); |
95 | |
96 | static int irq_type = IRQ_TYPE_MSI; |
97 | module_param(irq_type, int, 0444); |
98 | MODULE_PARM_DESC(irq_type, "IRQ mode selection in pci_endpoint_test (0 - Legacy, 1 - MSI, 2 - MSI-X)" ); |
99 | |
100 | enum pci_barno { |
101 | BAR_0, |
102 | BAR_1, |
103 | BAR_2, |
104 | BAR_3, |
105 | BAR_4, |
106 | BAR_5, |
107 | }; |
108 | |
109 | struct pci_endpoint_test { |
110 | struct pci_dev *pdev; |
111 | void __iomem *base; |
112 | void __iomem *bar[PCI_STD_NUM_BARS]; |
113 | struct completion irq_raised; |
114 | int last_irq; |
115 | int num_irqs; |
116 | int irq_type; |
117 | /* mutex to protect the ioctls */ |
118 | struct mutex mutex; |
119 | struct miscdevice miscdev; |
120 | enum pci_barno test_reg_bar; |
121 | size_t alignment; |
122 | const char *name; |
123 | }; |
124 | |
125 | struct pci_endpoint_test_data { |
126 | enum pci_barno test_reg_bar; |
127 | size_t alignment; |
128 | int irq_type; |
129 | }; |
130 | |
131 | static inline u32 pci_endpoint_test_readl(struct pci_endpoint_test *test, |
132 | u32 offset) |
133 | { |
134 | return readl(addr: test->base + offset); |
135 | } |
136 | |
137 | static inline void pci_endpoint_test_writel(struct pci_endpoint_test *test, |
138 | u32 offset, u32 value) |
139 | { |
140 | writel(val: value, addr: test->base + offset); |
141 | } |
142 | |
143 | static inline u32 pci_endpoint_test_bar_readl(struct pci_endpoint_test *test, |
144 | int bar, int offset) |
145 | { |
146 | return readl(addr: test->bar[bar] + offset); |
147 | } |
148 | |
149 | static inline void pci_endpoint_test_bar_writel(struct pci_endpoint_test *test, |
150 | int bar, u32 offset, u32 value) |
151 | { |
152 | writel(val: value, addr: test->bar[bar] + offset); |
153 | } |
154 | |
155 | static irqreturn_t pci_endpoint_test_irqhandler(int irq, void *dev_id) |
156 | { |
157 | struct pci_endpoint_test *test = dev_id; |
158 | u32 reg; |
159 | |
160 | reg = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_STATUS); |
161 | if (reg & STATUS_IRQ_RAISED) { |
162 | test->last_irq = irq; |
163 | complete(&test->irq_raised); |
164 | } |
165 | |
166 | return IRQ_HANDLED; |
167 | } |
168 | |
169 | static void pci_endpoint_test_free_irq_vectors(struct pci_endpoint_test *test) |
170 | { |
171 | struct pci_dev *pdev = test->pdev; |
172 | |
173 | pci_free_irq_vectors(dev: pdev); |
174 | test->irq_type = IRQ_TYPE_UNDEFINED; |
175 | } |
176 | |
177 | static bool pci_endpoint_test_alloc_irq_vectors(struct pci_endpoint_test *test, |
178 | int type) |
179 | { |
180 | int irq = -1; |
181 | struct pci_dev *pdev = test->pdev; |
182 | struct device *dev = &pdev->dev; |
183 | bool res = true; |
184 | |
185 | switch (type) { |
186 | case IRQ_TYPE_LEGACY: |
187 | irq = pci_alloc_irq_vectors(dev: pdev, min_vecs: 1, max_vecs: 1, PCI_IRQ_LEGACY); |
188 | if (irq < 0) |
189 | dev_err(dev, "Failed to get Legacy interrupt\n" ); |
190 | break; |
191 | case IRQ_TYPE_MSI: |
192 | irq = pci_alloc_irq_vectors(dev: pdev, min_vecs: 1, max_vecs: 32, PCI_IRQ_MSI); |
193 | if (irq < 0) |
194 | dev_err(dev, "Failed to get MSI interrupts\n" ); |
195 | break; |
196 | case IRQ_TYPE_MSIX: |
197 | irq = pci_alloc_irq_vectors(dev: pdev, min_vecs: 1, max_vecs: 2048, PCI_IRQ_MSIX); |
198 | if (irq < 0) |
199 | dev_err(dev, "Failed to get MSI-X interrupts\n" ); |
200 | break; |
201 | default: |
202 | dev_err(dev, "Invalid IRQ type selected\n" ); |
203 | } |
204 | |
205 | if (irq < 0) { |
206 | irq = 0; |
207 | res = false; |
208 | } |
209 | |
210 | test->irq_type = type; |
211 | test->num_irqs = irq; |
212 | |
213 | return res; |
214 | } |
215 | |
216 | static void pci_endpoint_test_release_irq(struct pci_endpoint_test *test) |
217 | { |
218 | int i; |
219 | struct pci_dev *pdev = test->pdev; |
220 | struct device *dev = &pdev->dev; |
221 | |
222 | for (i = 0; i < test->num_irqs; i++) |
223 | devm_free_irq(dev, irq: pci_irq_vector(dev: pdev, nr: i), dev_id: test); |
224 | |
225 | test->num_irqs = 0; |
226 | } |
227 | |
228 | static bool pci_endpoint_test_request_irq(struct pci_endpoint_test *test) |
229 | { |
230 | int i; |
231 | int err; |
232 | struct pci_dev *pdev = test->pdev; |
233 | struct device *dev = &pdev->dev; |
234 | |
235 | for (i = 0; i < test->num_irqs; i++) { |
236 | err = devm_request_irq(dev, irq: pci_irq_vector(dev: pdev, nr: i), |
237 | handler: pci_endpoint_test_irqhandler, |
238 | IRQF_SHARED, devname: test->name, dev_id: test); |
239 | if (err) |
240 | goto fail; |
241 | } |
242 | |
243 | return true; |
244 | |
245 | fail: |
246 | switch (irq_type) { |
247 | case IRQ_TYPE_LEGACY: |
248 | dev_err(dev, "Failed to request IRQ %d for Legacy\n" , |
249 | pci_irq_vector(pdev, i)); |
250 | break; |
251 | case IRQ_TYPE_MSI: |
252 | dev_err(dev, "Failed to request IRQ %d for MSI %d\n" , |
253 | pci_irq_vector(pdev, i), |
254 | i + 1); |
255 | break; |
256 | case IRQ_TYPE_MSIX: |
257 | dev_err(dev, "Failed to request IRQ %d for MSI-X %d\n" , |
258 | pci_irq_vector(pdev, i), |
259 | i + 1); |
260 | break; |
261 | } |
262 | |
263 | return false; |
264 | } |
265 | |
266 | static bool pci_endpoint_test_bar(struct pci_endpoint_test *test, |
267 | enum pci_barno barno) |
268 | { |
269 | int j; |
270 | u32 val; |
271 | int size; |
272 | struct pci_dev *pdev = test->pdev; |
273 | |
274 | if (!test->bar[barno]) |
275 | return false; |
276 | |
277 | size = pci_resource_len(pdev, barno); |
278 | |
279 | if (barno == test->test_reg_bar) |
280 | size = 0x4; |
281 | |
282 | for (j = 0; j < size; j += 4) |
283 | pci_endpoint_test_bar_writel(test, bar: barno, offset: j, value: 0xA0A0A0A0); |
284 | |
285 | for (j = 0; j < size; j += 4) { |
286 | val = pci_endpoint_test_bar_readl(test, bar: barno, offset: j); |
287 | if (val != 0xA0A0A0A0) |
288 | return false; |
289 | } |
290 | |
291 | return true; |
292 | } |
293 | |
294 | static bool pci_endpoint_test_legacy_irq(struct pci_endpoint_test *test) |
295 | { |
296 | u32 val; |
297 | |
298 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE, |
299 | IRQ_TYPE_LEGACY); |
300 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, value: 0); |
301 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND, |
302 | COMMAND_RAISE_LEGACY_IRQ); |
303 | val = wait_for_completion_timeout(x: &test->irq_raised, |
304 | timeout: msecs_to_jiffies(m: 1000)); |
305 | if (!val) |
306 | return false; |
307 | |
308 | return true; |
309 | } |
310 | |
311 | static bool pci_endpoint_test_msi_irq(struct pci_endpoint_test *test, |
312 | u16 msi_num, bool msix) |
313 | { |
314 | u32 val; |
315 | struct pci_dev *pdev = test->pdev; |
316 | |
317 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE, |
318 | value: msix ? IRQ_TYPE_MSIX : IRQ_TYPE_MSI); |
319 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, value: msi_num); |
320 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND, |
321 | value: msix ? COMMAND_RAISE_MSIX_IRQ : |
322 | COMMAND_RAISE_MSI_IRQ); |
323 | val = wait_for_completion_timeout(x: &test->irq_raised, |
324 | timeout: msecs_to_jiffies(m: 1000)); |
325 | if (!val) |
326 | return false; |
327 | |
328 | return pci_irq_vector(dev: pdev, nr: msi_num - 1) == test->last_irq; |
329 | } |
330 | |
331 | static int pci_endpoint_test_validate_xfer_params(struct device *dev, |
332 | struct pci_endpoint_test_xfer_param *param, size_t alignment) |
333 | { |
334 | if (!param->size) { |
335 | dev_dbg(dev, "Data size is zero\n" ); |
336 | return -EINVAL; |
337 | } |
338 | |
339 | if (param->size > SIZE_MAX - alignment) { |
340 | dev_dbg(dev, "Maximum transfer data size exceeded\n" ); |
341 | return -EINVAL; |
342 | } |
343 | |
344 | return 0; |
345 | } |
346 | |
347 | static bool pci_endpoint_test_copy(struct pci_endpoint_test *test, |
348 | unsigned long arg) |
349 | { |
350 | struct pci_endpoint_test_xfer_param param; |
351 | bool ret = false; |
352 | void *src_addr; |
353 | void *dst_addr; |
354 | u32 flags = 0; |
355 | bool use_dma; |
356 | size_t size; |
357 | dma_addr_t src_phys_addr; |
358 | dma_addr_t dst_phys_addr; |
359 | struct pci_dev *pdev = test->pdev; |
360 | struct device *dev = &pdev->dev; |
361 | void *orig_src_addr; |
362 | dma_addr_t orig_src_phys_addr; |
363 | void *orig_dst_addr; |
364 | dma_addr_t orig_dst_phys_addr; |
365 | size_t offset; |
366 | size_t alignment = test->alignment; |
367 | int irq_type = test->irq_type; |
368 | u32 src_crc32; |
369 | u32 dst_crc32; |
370 | int err; |
371 | |
372 | err = copy_from_user(to: ¶m, from: (void __user *)arg, n: sizeof(param)); |
373 | if (err) { |
374 | dev_err(dev, "Failed to get transfer param\n" ); |
375 | return false; |
376 | } |
377 | |
378 | err = pci_endpoint_test_validate_xfer_params(dev, param: ¶m, alignment); |
379 | if (err) |
380 | return false; |
381 | |
382 | size = param.size; |
383 | |
384 | use_dma = !!(param.flags & PCITEST_FLAGS_USE_DMA); |
385 | if (use_dma) |
386 | flags |= FLAG_USE_DMA; |
387 | |
388 | if (irq_type < IRQ_TYPE_LEGACY || irq_type > IRQ_TYPE_MSIX) { |
389 | dev_err(dev, "Invalid IRQ type option\n" ); |
390 | goto err; |
391 | } |
392 | |
393 | orig_src_addr = kzalloc(size: size + alignment, GFP_KERNEL); |
394 | if (!orig_src_addr) { |
395 | dev_err(dev, "Failed to allocate source buffer\n" ); |
396 | ret = false; |
397 | goto err; |
398 | } |
399 | |
400 | get_random_bytes(buf: orig_src_addr, len: size + alignment); |
401 | orig_src_phys_addr = dma_map_single(dev, orig_src_addr, |
402 | size + alignment, DMA_TO_DEVICE); |
403 | if (dma_mapping_error(dev, dma_addr: orig_src_phys_addr)) { |
404 | dev_err(dev, "failed to map source buffer address\n" ); |
405 | ret = false; |
406 | goto err_src_phys_addr; |
407 | } |
408 | |
409 | if (alignment && !IS_ALIGNED(orig_src_phys_addr, alignment)) { |
410 | src_phys_addr = PTR_ALIGN(orig_src_phys_addr, alignment); |
411 | offset = src_phys_addr - orig_src_phys_addr; |
412 | src_addr = orig_src_addr + offset; |
413 | } else { |
414 | src_phys_addr = orig_src_phys_addr; |
415 | src_addr = orig_src_addr; |
416 | } |
417 | |
418 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_LOWER_SRC_ADDR, |
419 | lower_32_bits(src_phys_addr)); |
420 | |
421 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_UPPER_SRC_ADDR, |
422 | upper_32_bits(src_phys_addr)); |
423 | |
424 | src_crc32 = crc32_le(crc: ~0, p: src_addr, len: size); |
425 | |
426 | orig_dst_addr = kzalloc(size: size + alignment, GFP_KERNEL); |
427 | if (!orig_dst_addr) { |
428 | dev_err(dev, "Failed to allocate destination address\n" ); |
429 | ret = false; |
430 | goto err_dst_addr; |
431 | } |
432 | |
433 | orig_dst_phys_addr = dma_map_single(dev, orig_dst_addr, |
434 | size + alignment, DMA_FROM_DEVICE); |
435 | if (dma_mapping_error(dev, dma_addr: orig_dst_phys_addr)) { |
436 | dev_err(dev, "failed to map destination buffer address\n" ); |
437 | ret = false; |
438 | goto err_dst_phys_addr; |
439 | } |
440 | |
441 | if (alignment && !IS_ALIGNED(orig_dst_phys_addr, alignment)) { |
442 | dst_phys_addr = PTR_ALIGN(orig_dst_phys_addr, alignment); |
443 | offset = dst_phys_addr - orig_dst_phys_addr; |
444 | dst_addr = orig_dst_addr + offset; |
445 | } else { |
446 | dst_phys_addr = orig_dst_phys_addr; |
447 | dst_addr = orig_dst_addr; |
448 | } |
449 | |
450 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_LOWER_DST_ADDR, |
451 | lower_32_bits(dst_phys_addr)); |
452 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_UPPER_DST_ADDR, |
453 | upper_32_bits(dst_phys_addr)); |
454 | |
455 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_SIZE, |
456 | value: size); |
457 | |
458 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_FLAGS, value: flags); |
459 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE, value: irq_type); |
460 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, value: 1); |
461 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND, |
462 | COMMAND_COPY); |
463 | |
464 | wait_for_completion(&test->irq_raised); |
465 | |
466 | dma_unmap_single(dev, orig_dst_phys_addr, size + alignment, |
467 | DMA_FROM_DEVICE); |
468 | |
469 | dst_crc32 = crc32_le(crc: ~0, p: dst_addr, len: size); |
470 | if (dst_crc32 == src_crc32) |
471 | ret = true; |
472 | |
473 | err_dst_phys_addr: |
474 | kfree(objp: orig_dst_addr); |
475 | |
476 | err_dst_addr: |
477 | dma_unmap_single(dev, orig_src_phys_addr, size + alignment, |
478 | DMA_TO_DEVICE); |
479 | |
480 | err_src_phys_addr: |
481 | kfree(objp: orig_src_addr); |
482 | |
483 | err: |
484 | return ret; |
485 | } |
486 | |
487 | static bool pci_endpoint_test_write(struct pci_endpoint_test *test, |
488 | unsigned long arg) |
489 | { |
490 | struct pci_endpoint_test_xfer_param param; |
491 | bool ret = false; |
492 | u32 flags = 0; |
493 | bool use_dma; |
494 | u32 reg; |
495 | void *addr; |
496 | dma_addr_t phys_addr; |
497 | struct pci_dev *pdev = test->pdev; |
498 | struct device *dev = &pdev->dev; |
499 | void *orig_addr; |
500 | dma_addr_t orig_phys_addr; |
501 | size_t offset; |
502 | size_t alignment = test->alignment; |
503 | int irq_type = test->irq_type; |
504 | size_t size; |
505 | u32 crc32; |
506 | int err; |
507 | |
508 | err = copy_from_user(to: ¶m, from: (void __user *)arg, n: sizeof(param)); |
509 | if (err != 0) { |
510 | dev_err(dev, "Failed to get transfer param\n" ); |
511 | return false; |
512 | } |
513 | |
514 | err = pci_endpoint_test_validate_xfer_params(dev, param: ¶m, alignment); |
515 | if (err) |
516 | return false; |
517 | |
518 | size = param.size; |
519 | |
520 | use_dma = !!(param.flags & PCITEST_FLAGS_USE_DMA); |
521 | if (use_dma) |
522 | flags |= FLAG_USE_DMA; |
523 | |
524 | if (irq_type < IRQ_TYPE_LEGACY || irq_type > IRQ_TYPE_MSIX) { |
525 | dev_err(dev, "Invalid IRQ type option\n" ); |
526 | goto err; |
527 | } |
528 | |
529 | orig_addr = kzalloc(size: size + alignment, GFP_KERNEL); |
530 | if (!orig_addr) { |
531 | dev_err(dev, "Failed to allocate address\n" ); |
532 | ret = false; |
533 | goto err; |
534 | } |
535 | |
536 | get_random_bytes(buf: orig_addr, len: size + alignment); |
537 | |
538 | orig_phys_addr = dma_map_single(dev, orig_addr, size + alignment, |
539 | DMA_TO_DEVICE); |
540 | if (dma_mapping_error(dev, dma_addr: orig_phys_addr)) { |
541 | dev_err(dev, "failed to map source buffer address\n" ); |
542 | ret = false; |
543 | goto err_phys_addr; |
544 | } |
545 | |
546 | if (alignment && !IS_ALIGNED(orig_phys_addr, alignment)) { |
547 | phys_addr = PTR_ALIGN(orig_phys_addr, alignment); |
548 | offset = phys_addr - orig_phys_addr; |
549 | addr = orig_addr + offset; |
550 | } else { |
551 | phys_addr = orig_phys_addr; |
552 | addr = orig_addr; |
553 | } |
554 | |
555 | crc32 = crc32_le(crc: ~0, p: addr, len: size); |
556 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_CHECKSUM, |
557 | value: crc32); |
558 | |
559 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_LOWER_SRC_ADDR, |
560 | lower_32_bits(phys_addr)); |
561 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_UPPER_SRC_ADDR, |
562 | upper_32_bits(phys_addr)); |
563 | |
564 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_SIZE, value: size); |
565 | |
566 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_FLAGS, value: flags); |
567 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE, value: irq_type); |
568 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, value: 1); |
569 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND, |
570 | COMMAND_READ); |
571 | |
572 | wait_for_completion(&test->irq_raised); |
573 | |
574 | reg = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_STATUS); |
575 | if (reg & STATUS_READ_SUCCESS) |
576 | ret = true; |
577 | |
578 | dma_unmap_single(dev, orig_phys_addr, size + alignment, |
579 | DMA_TO_DEVICE); |
580 | |
581 | err_phys_addr: |
582 | kfree(objp: orig_addr); |
583 | |
584 | err: |
585 | return ret; |
586 | } |
587 | |
588 | static bool pci_endpoint_test_read(struct pci_endpoint_test *test, |
589 | unsigned long arg) |
590 | { |
591 | struct pci_endpoint_test_xfer_param param; |
592 | bool ret = false; |
593 | u32 flags = 0; |
594 | bool use_dma; |
595 | size_t size; |
596 | void *addr; |
597 | dma_addr_t phys_addr; |
598 | struct pci_dev *pdev = test->pdev; |
599 | struct device *dev = &pdev->dev; |
600 | void *orig_addr; |
601 | dma_addr_t orig_phys_addr; |
602 | size_t offset; |
603 | size_t alignment = test->alignment; |
604 | int irq_type = test->irq_type; |
605 | u32 crc32; |
606 | int err; |
607 | |
608 | err = copy_from_user(to: ¶m, from: (void __user *)arg, n: sizeof(param)); |
609 | if (err) { |
610 | dev_err(dev, "Failed to get transfer param\n" ); |
611 | return false; |
612 | } |
613 | |
614 | err = pci_endpoint_test_validate_xfer_params(dev, param: ¶m, alignment); |
615 | if (err) |
616 | return false; |
617 | |
618 | size = param.size; |
619 | |
620 | use_dma = !!(param.flags & PCITEST_FLAGS_USE_DMA); |
621 | if (use_dma) |
622 | flags |= FLAG_USE_DMA; |
623 | |
624 | if (irq_type < IRQ_TYPE_LEGACY || irq_type > IRQ_TYPE_MSIX) { |
625 | dev_err(dev, "Invalid IRQ type option\n" ); |
626 | goto err; |
627 | } |
628 | |
629 | orig_addr = kzalloc(size: size + alignment, GFP_KERNEL); |
630 | if (!orig_addr) { |
631 | dev_err(dev, "Failed to allocate destination address\n" ); |
632 | ret = false; |
633 | goto err; |
634 | } |
635 | |
636 | orig_phys_addr = dma_map_single(dev, orig_addr, size + alignment, |
637 | DMA_FROM_DEVICE); |
638 | if (dma_mapping_error(dev, dma_addr: orig_phys_addr)) { |
639 | dev_err(dev, "failed to map source buffer address\n" ); |
640 | ret = false; |
641 | goto err_phys_addr; |
642 | } |
643 | |
644 | if (alignment && !IS_ALIGNED(orig_phys_addr, alignment)) { |
645 | phys_addr = PTR_ALIGN(orig_phys_addr, alignment); |
646 | offset = phys_addr - orig_phys_addr; |
647 | addr = orig_addr + offset; |
648 | } else { |
649 | phys_addr = orig_phys_addr; |
650 | addr = orig_addr; |
651 | } |
652 | |
653 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_LOWER_DST_ADDR, |
654 | lower_32_bits(phys_addr)); |
655 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_UPPER_DST_ADDR, |
656 | upper_32_bits(phys_addr)); |
657 | |
658 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_SIZE, value: size); |
659 | |
660 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_FLAGS, value: flags); |
661 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE, value: irq_type); |
662 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, value: 1); |
663 | pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND, |
664 | COMMAND_WRITE); |
665 | |
666 | wait_for_completion(&test->irq_raised); |
667 | |
668 | dma_unmap_single(dev, orig_phys_addr, size + alignment, |
669 | DMA_FROM_DEVICE); |
670 | |
671 | crc32 = crc32_le(crc: ~0, p: addr, len: size); |
672 | if (crc32 == pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_CHECKSUM)) |
673 | ret = true; |
674 | |
675 | err_phys_addr: |
676 | kfree(objp: orig_addr); |
677 | err: |
678 | return ret; |
679 | } |
680 | |
681 | static bool pci_endpoint_test_clear_irq(struct pci_endpoint_test *test) |
682 | { |
683 | pci_endpoint_test_release_irq(test); |
684 | pci_endpoint_test_free_irq_vectors(test); |
685 | return true; |
686 | } |
687 | |
688 | static bool pci_endpoint_test_set_irq(struct pci_endpoint_test *test, |
689 | int req_irq_type) |
690 | { |
691 | struct pci_dev *pdev = test->pdev; |
692 | struct device *dev = &pdev->dev; |
693 | |
694 | if (req_irq_type < IRQ_TYPE_LEGACY || req_irq_type > IRQ_TYPE_MSIX) { |
695 | dev_err(dev, "Invalid IRQ type option\n" ); |
696 | return false; |
697 | } |
698 | |
699 | if (test->irq_type == req_irq_type) |
700 | return true; |
701 | |
702 | pci_endpoint_test_release_irq(test); |
703 | pci_endpoint_test_free_irq_vectors(test); |
704 | |
705 | if (!pci_endpoint_test_alloc_irq_vectors(test, type: req_irq_type)) |
706 | goto err; |
707 | |
708 | if (!pci_endpoint_test_request_irq(test)) |
709 | goto err; |
710 | |
711 | return true; |
712 | |
713 | err: |
714 | pci_endpoint_test_free_irq_vectors(test); |
715 | return false; |
716 | } |
717 | |
718 | static long pci_endpoint_test_ioctl(struct file *file, unsigned int cmd, |
719 | unsigned long arg) |
720 | { |
721 | int ret = -EINVAL; |
722 | enum pci_barno bar; |
723 | struct pci_endpoint_test *test = to_endpoint_test(file->private_data); |
724 | struct pci_dev *pdev = test->pdev; |
725 | |
726 | mutex_lock(&test->mutex); |
727 | |
728 | reinit_completion(x: &test->irq_raised); |
729 | test->last_irq = -ENODATA; |
730 | |
731 | switch (cmd) { |
732 | case PCITEST_BAR: |
733 | bar = arg; |
734 | if (bar > BAR_5) |
735 | goto ret; |
736 | if (is_am654_pci_dev(pdev) && bar == BAR_0) |
737 | goto ret; |
738 | ret = pci_endpoint_test_bar(test, barno: bar); |
739 | break; |
740 | case PCITEST_LEGACY_IRQ: |
741 | ret = pci_endpoint_test_legacy_irq(test); |
742 | break; |
743 | case PCITEST_MSI: |
744 | case PCITEST_MSIX: |
745 | ret = pci_endpoint_test_msi_irq(test, msi_num: arg, msix: cmd == PCITEST_MSIX); |
746 | break; |
747 | case PCITEST_WRITE: |
748 | ret = pci_endpoint_test_write(test, arg); |
749 | break; |
750 | case PCITEST_READ: |
751 | ret = pci_endpoint_test_read(test, arg); |
752 | break; |
753 | case PCITEST_COPY: |
754 | ret = pci_endpoint_test_copy(test, arg); |
755 | break; |
756 | case PCITEST_SET_IRQTYPE: |
757 | ret = pci_endpoint_test_set_irq(test, req_irq_type: arg); |
758 | break; |
759 | case PCITEST_GET_IRQTYPE: |
760 | ret = irq_type; |
761 | break; |
762 | case PCITEST_CLEAR_IRQ: |
763 | ret = pci_endpoint_test_clear_irq(test); |
764 | break; |
765 | } |
766 | |
767 | ret: |
768 | mutex_unlock(lock: &test->mutex); |
769 | return ret; |
770 | } |
771 | |
772 | static const struct file_operations pci_endpoint_test_fops = { |
773 | .owner = THIS_MODULE, |
774 | .unlocked_ioctl = pci_endpoint_test_ioctl, |
775 | }; |
776 | |
777 | static int pci_endpoint_test_probe(struct pci_dev *pdev, |
778 | const struct pci_device_id *ent) |
779 | { |
780 | int err; |
781 | int id; |
782 | char name[24]; |
783 | enum pci_barno bar; |
784 | void __iomem *base; |
785 | struct device *dev = &pdev->dev; |
786 | struct pci_endpoint_test *test; |
787 | struct pci_endpoint_test_data *data; |
788 | enum pci_barno test_reg_bar = BAR_0; |
789 | struct miscdevice *misc_device; |
790 | |
791 | if (pci_is_bridge(dev: pdev)) |
792 | return -ENODEV; |
793 | |
794 | test = devm_kzalloc(dev, size: sizeof(*test), GFP_KERNEL); |
795 | if (!test) |
796 | return -ENOMEM; |
797 | |
798 | test->test_reg_bar = 0; |
799 | test->alignment = 0; |
800 | test->pdev = pdev; |
801 | test->irq_type = IRQ_TYPE_UNDEFINED; |
802 | |
803 | if (no_msi) |
804 | irq_type = IRQ_TYPE_LEGACY; |
805 | |
806 | data = (struct pci_endpoint_test_data *)ent->driver_data; |
807 | if (data) { |
808 | test_reg_bar = data->test_reg_bar; |
809 | test->test_reg_bar = test_reg_bar; |
810 | test->alignment = data->alignment; |
811 | irq_type = data->irq_type; |
812 | } |
813 | |
814 | init_completion(x: &test->irq_raised); |
815 | mutex_init(&test->mutex); |
816 | |
817 | if ((dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(48)) != 0) && |
818 | dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(32)) != 0) { |
819 | dev_err(dev, "Cannot set DMA mask\n" ); |
820 | return -EINVAL; |
821 | } |
822 | |
823 | err = pci_enable_device(dev: pdev); |
824 | if (err) { |
825 | dev_err(dev, "Cannot enable PCI device\n" ); |
826 | return err; |
827 | } |
828 | |
829 | err = pci_request_regions(pdev, DRV_MODULE_NAME); |
830 | if (err) { |
831 | dev_err(dev, "Cannot obtain PCI resources\n" ); |
832 | goto err_disable_pdev; |
833 | } |
834 | |
835 | pci_set_master(dev: pdev); |
836 | |
837 | if (!pci_endpoint_test_alloc_irq_vectors(test, type: irq_type)) { |
838 | err = -EINVAL; |
839 | goto err_disable_irq; |
840 | } |
841 | |
842 | for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) { |
843 | if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) { |
844 | base = pci_ioremap_bar(pdev, bar); |
845 | if (!base) { |
846 | dev_err(dev, "Failed to read BAR%d\n" , bar); |
847 | WARN_ON(bar == test_reg_bar); |
848 | } |
849 | test->bar[bar] = base; |
850 | } |
851 | } |
852 | |
853 | test->base = test->bar[test_reg_bar]; |
854 | if (!test->base) { |
855 | err = -ENOMEM; |
856 | dev_err(dev, "Cannot perform PCI test without BAR%d\n" , |
857 | test_reg_bar); |
858 | goto err_iounmap; |
859 | } |
860 | |
861 | pci_set_drvdata(pdev, data: test); |
862 | |
863 | id = ida_simple_get(&pci_endpoint_test_ida, 0, 0, GFP_KERNEL); |
864 | if (id < 0) { |
865 | err = id; |
866 | dev_err(dev, "Unable to get id\n" ); |
867 | goto err_iounmap; |
868 | } |
869 | |
870 | snprintf(buf: name, size: sizeof(name), DRV_MODULE_NAME ".%d" , id); |
871 | test->name = kstrdup(s: name, GFP_KERNEL); |
872 | if (!test->name) { |
873 | err = -ENOMEM; |
874 | goto err_ida_remove; |
875 | } |
876 | |
877 | if (!pci_endpoint_test_request_irq(test)) { |
878 | err = -EINVAL; |
879 | goto err_kfree_test_name; |
880 | } |
881 | |
882 | misc_device = &test->miscdev; |
883 | misc_device->minor = MISC_DYNAMIC_MINOR; |
884 | misc_device->name = kstrdup(s: name, GFP_KERNEL); |
885 | if (!misc_device->name) { |
886 | err = -ENOMEM; |
887 | goto err_release_irq; |
888 | } |
889 | misc_device->parent = &pdev->dev; |
890 | misc_device->fops = &pci_endpoint_test_fops; |
891 | |
892 | err = misc_register(misc: misc_device); |
893 | if (err) { |
894 | dev_err(dev, "Failed to register device\n" ); |
895 | goto err_kfree_name; |
896 | } |
897 | |
898 | return 0; |
899 | |
900 | err_kfree_name: |
901 | kfree(objp: misc_device->name); |
902 | |
903 | err_release_irq: |
904 | pci_endpoint_test_release_irq(test); |
905 | |
906 | err_kfree_test_name: |
907 | kfree(objp: test->name); |
908 | |
909 | err_ida_remove: |
910 | ida_simple_remove(&pci_endpoint_test_ida, id); |
911 | |
912 | err_iounmap: |
913 | for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) { |
914 | if (test->bar[bar]) |
915 | pci_iounmap(dev: pdev, test->bar[bar]); |
916 | } |
917 | |
918 | err_disable_irq: |
919 | pci_endpoint_test_free_irq_vectors(test); |
920 | pci_release_regions(pdev); |
921 | |
922 | err_disable_pdev: |
923 | pci_disable_device(dev: pdev); |
924 | |
925 | return err; |
926 | } |
927 | |
928 | static void pci_endpoint_test_remove(struct pci_dev *pdev) |
929 | { |
930 | int id; |
931 | enum pci_barno bar; |
932 | struct pci_endpoint_test *test = pci_get_drvdata(pdev); |
933 | struct miscdevice *misc_device = &test->miscdev; |
934 | |
935 | if (sscanf(misc_device->name, DRV_MODULE_NAME ".%d" , &id) != 1) |
936 | return; |
937 | if (id < 0) |
938 | return; |
939 | |
940 | pci_endpoint_test_release_irq(test); |
941 | pci_endpoint_test_free_irq_vectors(test); |
942 | |
943 | misc_deregister(misc: &test->miscdev); |
944 | kfree(objp: misc_device->name); |
945 | kfree(objp: test->name); |
946 | ida_simple_remove(&pci_endpoint_test_ida, id); |
947 | for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) { |
948 | if (test->bar[bar]) |
949 | pci_iounmap(dev: pdev, test->bar[bar]); |
950 | } |
951 | |
952 | pci_release_regions(pdev); |
953 | pci_disable_device(dev: pdev); |
954 | } |
955 | |
956 | static const struct pci_endpoint_test_data default_data = { |
957 | .test_reg_bar = BAR_0, |
958 | .alignment = SZ_4K, |
959 | .irq_type = IRQ_TYPE_MSI, |
960 | }; |
961 | |
962 | static const struct pci_endpoint_test_data am654_data = { |
963 | .test_reg_bar = BAR_2, |
964 | .alignment = SZ_64K, |
965 | .irq_type = IRQ_TYPE_MSI, |
966 | }; |
967 | |
968 | static const struct pci_endpoint_test_data j721e_data = { |
969 | .alignment = 256, |
970 | .irq_type = IRQ_TYPE_MSI, |
971 | }; |
972 | |
973 | static const struct pci_device_id pci_endpoint_test_tbl[] = { |
974 | { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_DRA74x), |
975 | .driver_data = (kernel_ulong_t)&default_data, |
976 | }, |
977 | { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_DRA72x), |
978 | .driver_data = (kernel_ulong_t)&default_data, |
979 | }, |
980 | { PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, 0x81c0), |
981 | .driver_data = (kernel_ulong_t)&default_data, |
982 | }, |
983 | { PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, PCI_DEVICE_ID_IMX8),}, |
984 | { PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, PCI_DEVICE_ID_LS1088A), |
985 | .driver_data = (kernel_ulong_t)&default_data, |
986 | }, |
987 | { PCI_DEVICE_DATA(SYNOPSYS, EDDA, NULL) }, |
988 | { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_AM654), |
989 | .driver_data = (kernel_ulong_t)&am654_data |
990 | }, |
991 | { PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A774A1),}, |
992 | { PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A774B1),}, |
993 | { PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A774C0),}, |
994 | { PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A774E1),}, |
995 | { PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A779F0), |
996 | .driver_data = (kernel_ulong_t)&default_data, |
997 | }, |
998 | { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_J721E), |
999 | .driver_data = (kernel_ulong_t)&j721e_data, |
1000 | }, |
1001 | { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_J7200), |
1002 | .driver_data = (kernel_ulong_t)&j721e_data, |
1003 | }, |
1004 | { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_AM64), |
1005 | .driver_data = (kernel_ulong_t)&j721e_data, |
1006 | }, |
1007 | { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_J721S2), |
1008 | .driver_data = (kernel_ulong_t)&j721e_data, |
1009 | }, |
1010 | { } |
1011 | }; |
1012 | MODULE_DEVICE_TABLE(pci, pci_endpoint_test_tbl); |
1013 | |
1014 | static struct pci_driver pci_endpoint_test_driver = { |
1015 | .name = DRV_MODULE_NAME, |
1016 | .id_table = pci_endpoint_test_tbl, |
1017 | .probe = pci_endpoint_test_probe, |
1018 | .remove = pci_endpoint_test_remove, |
1019 | .sriov_configure = pci_sriov_configure_simple, |
1020 | }; |
1021 | module_pci_driver(pci_endpoint_test_driver); |
1022 | |
1023 | MODULE_DESCRIPTION("PCI ENDPOINT TEST HOST DRIVER" ); |
1024 | MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>" ); |
1025 | MODULE_LICENSE("GPL v2" ); |
1026 | |