1/*
2 * DMA Engine test module
3 *
4 * Copyright (C) 2007 Atmel Corporation
5 * Copyright (C) 2013 Intel Corporation
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12
13#include <linux/delay.h>
14#include <linux/dma-mapping.h>
15#include <linux/dmaengine.h>
16#include <linux/freezer.h>
17#include <linux/init.h>
18#include <linux/kthread.h>
19#include <linux/sched/task.h>
20#include <linux/module.h>
21#include <linux/moduleparam.h>
22#include <linux/random.h>
23#include <linux/slab.h>
24#include <linux/wait.h>
25
26static unsigned int test_buf_size = 16384;
27module_param(test_buf_size, uint, S_IRUGO | S_IWUSR);
28MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
29
30static char test_device[32];
31module_param_string(device, test_device, sizeof(test_device),
32 S_IRUGO | S_IWUSR);
33MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");
34
35static unsigned int threads_per_chan = 1;
36module_param(threads_per_chan, uint, S_IRUGO | S_IWUSR);
37MODULE_PARM_DESC(threads_per_chan,
38 "Number of threads to start per channel (default: 1)");
39
40static unsigned int max_channels;
41module_param(max_channels, uint, S_IRUGO | S_IWUSR);
42MODULE_PARM_DESC(max_channels,
43 "Maximum number of channels to use (default: all)");
44
45static unsigned int iterations;
46module_param(iterations, uint, S_IRUGO | S_IWUSR);
47MODULE_PARM_DESC(iterations,
48 "Iterations before stopping test (default: infinite)");
49
50static unsigned int dmatest;
51module_param(dmatest, uint, S_IRUGO | S_IWUSR);
52MODULE_PARM_DESC(dmatest,
53 "dmatest 0-memcpy 1-memset (default: 0)");
54
55static unsigned int xor_sources = 3;
56module_param(xor_sources, uint, S_IRUGO | S_IWUSR);
57MODULE_PARM_DESC(xor_sources,
58 "Number of xor source buffers (default: 3)");
59
60static unsigned int pq_sources = 3;
61module_param(pq_sources, uint, S_IRUGO | S_IWUSR);
62MODULE_PARM_DESC(pq_sources,
63 "Number of p+q source buffers (default: 3)");
64
65static int timeout = 3000;
66module_param(timeout, uint, S_IRUGO | S_IWUSR);
67MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
68 "Pass -1 for infinite timeout");
69
70static bool noverify;
71module_param(noverify, bool, S_IRUGO | S_IWUSR);
72MODULE_PARM_DESC(noverify, "Disable data verification (default: verify)");
73
74static bool norandom;
75module_param(norandom, bool, 0644);
76MODULE_PARM_DESC(norandom, "Disable random offset setup (default: random)");
77
78static bool verbose;
79module_param(verbose, bool, S_IRUGO | S_IWUSR);
80MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)");
81
82static int alignment = -1;
83module_param(alignment, int, 0644);
84MODULE_PARM_DESC(alignment, "Custom data address alignment taken as 2^(alignment) (default: not used (-1))");
85
86static unsigned int transfer_size;
87module_param(transfer_size, uint, 0644);
88MODULE_PARM_DESC(transfer_size, "Optional custom transfer size in bytes (default: not used (0))");
89
90/**
91 * struct dmatest_params - test parameters.
92 * @buf_size: size of the memcpy test buffer
93 * @channel: bus ID of the channel to test
94 * @device: bus ID of the DMA Engine to test
95 * @threads_per_chan: number of threads to start per channel
96 * @max_channels: maximum number of channels to use
97 * @iterations: iterations before stopping test
98 * @xor_sources: number of xor source buffers
99 * @pq_sources: number of p+q source buffers
100 * @timeout: transfer timeout in msec, -1 for infinite timeout
101 */
102struct dmatest_params {
103 unsigned int buf_size;
104 char channel[20];
105 char device[32];
106 unsigned int threads_per_chan;
107 unsigned int max_channels;
108 unsigned int iterations;
109 unsigned int xor_sources;
110 unsigned int pq_sources;
111 int timeout;
112 bool noverify;
113 bool norandom;
114 int alignment;
115 unsigned int transfer_size;
116};
117
118/**
119 * struct dmatest_info - test information.
120 * @params: test parameters
121 * @lock: access protection to the fields of this structure
122 */
123static struct dmatest_info {
124 /* Test parameters */
125 struct dmatest_params params;
126
127 /* Internal state */
128 struct list_head channels;
129 unsigned int nr_channels;
130 struct mutex lock;
131 bool did_init;
132} test_info = {
133 .channels = LIST_HEAD_INIT(test_info.channels),
134 .lock = __MUTEX_INITIALIZER(test_info.lock),
135};
136
137static int dmatest_run_set(const char *val, const struct kernel_param *kp);
138static int dmatest_run_get(char *val, const struct kernel_param *kp);
139static const struct kernel_param_ops run_ops = {
140 .set = dmatest_run_set,
141 .get = dmatest_run_get,
142};
143static bool dmatest_run;
144module_param_cb(run, &run_ops, &dmatest_run, S_IRUGO | S_IWUSR);
145MODULE_PARM_DESC(run, "Run the test (default: false)");
146
147static int dmatest_chan_set(const char *val, const struct kernel_param *kp);
148static int dmatest_chan_get(char *val, const struct kernel_param *kp);
149static const struct kernel_param_ops multi_chan_ops = {
150 .set = dmatest_chan_set,
151 .get = dmatest_chan_get,
152};
153
154static char test_channel[20];
155static struct kparam_string newchan_kps = {
156 .string = test_channel,
157 .maxlen = 20,
158};
159module_param_cb(channel, &multi_chan_ops, &newchan_kps, 0644);
160MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
161
162static int dmatest_test_list_get(char *val, const struct kernel_param *kp);
163static const struct kernel_param_ops test_list_ops = {
164 .get = dmatest_test_list_get,
165};
166module_param_cb(test_list, &test_list_ops, NULL, 0444);
167MODULE_PARM_DESC(test_list, "Print current test list");
168
169/* Maximum amount of mismatched bytes in buffer to print */
170#define MAX_ERROR_COUNT 32
171
172/*
173 * Initialization patterns. All bytes in the source buffer has bit 7
174 * set, all bytes in the destination buffer has bit 7 cleared.
175 *
176 * Bit 6 is set for all bytes which are to be copied by the DMA
177 * engine. Bit 5 is set for all bytes which are to be overwritten by
178 * the DMA engine.
179 *
180 * The remaining bits are the inverse of a counter which increments by
181 * one for each byte address.
182 */
183#define PATTERN_SRC 0x80
184#define PATTERN_DST 0x00
185#define PATTERN_COPY 0x40
186#define PATTERN_OVERWRITE 0x20
187#define PATTERN_COUNT_MASK 0x1f
188#define PATTERN_MEMSET_IDX 0x01
189
190/* Fixed point arithmetic ops */
191#define FIXPT_SHIFT 8
192#define FIXPNT_MASK 0xFF
193#define FIXPT_TO_INT(a) ((a) >> FIXPT_SHIFT)
194#define INT_TO_FIXPT(a) ((a) << FIXPT_SHIFT)
195#define FIXPT_GET_FRAC(a) ((((a) & FIXPNT_MASK) * 100) >> FIXPT_SHIFT)
196
197/* poor man's completion - we want to use wait_event_freezable() on it */
198struct dmatest_done {
199 bool done;
200 wait_queue_head_t *wait;
201};
202
203struct dmatest_data {
204 u8 **raw;
205 u8 **aligned;
206 unsigned int cnt;
207 unsigned int off;
208};
209
210struct dmatest_thread {
211 struct list_head node;
212 struct dmatest_info *info;
213 struct task_struct *task;
214 struct dma_chan *chan;
215 struct dmatest_data src;
216 struct dmatest_data dst;
217 enum dma_transaction_type type;
218 wait_queue_head_t done_wait;
219 struct dmatest_done test_done;
220 bool done;
221 bool pending;
222};
223
224struct dmatest_chan {
225 struct list_head node;
226 struct dma_chan *chan;
227 struct list_head threads;
228};
229
230static DECLARE_WAIT_QUEUE_HEAD(thread_wait);
231static bool wait;
232
233static bool is_threaded_test_run(struct dmatest_info *info)
234{
235 struct dmatest_chan *dtc;
236
237 list_for_each_entry(dtc, &info->channels, node) {
238 struct dmatest_thread *thread;
239
240 list_for_each_entry(thread, &dtc->threads, node) {
241 if (!thread->done)
242 return true;
243 }
244 }
245
246 return false;
247}
248
249static bool is_threaded_test_pending(struct dmatest_info *info)
250{
251 struct dmatest_chan *dtc;
252
253 list_for_each_entry(dtc, &info->channels, node) {
254 struct dmatest_thread *thread;
255
256 list_for_each_entry(thread, &dtc->threads, node) {
257 if (thread->pending)
258 return true;
259 }
260 }
261
262 return false;
263}
264
265static int dmatest_wait_get(char *val, const struct kernel_param *kp)
266{
267 struct dmatest_info *info = &test_info;
268 struct dmatest_params *params = &info->params;
269
270 if (params->iterations)
271 wait_event(thread_wait, !is_threaded_test_run(info));
272 wait = true;
273 return param_get_bool(val, kp);
274}
275
276static const struct kernel_param_ops wait_ops = {
277 .get = dmatest_wait_get,
278 .set = param_set_bool,
279};
280module_param_cb(wait, &wait_ops, &wait, S_IRUGO);
281MODULE_PARM_DESC(wait, "Wait for tests to complete (default: false)");
282
283static bool dmatest_match_channel(struct dmatest_params *params,
284 struct dma_chan *chan)
285{
286 if (params->channel[0] == '\0')
287 return true;
288 return strcmp(dma_chan_name(chan), params->channel) == 0;
289}
290
291static bool dmatest_match_device(struct dmatest_params *params,
292 struct dma_device *device)
293{
294 if (params->device[0] == '\0')
295 return true;
296 return strcmp(dev_name(device->dev), params->device) == 0;
297}
298
299static unsigned long dmatest_random(void)
300{
301 unsigned long buf;
302
303 prandom_bytes(&buf, sizeof(buf));
304 return buf;
305}
306
307static inline u8 gen_inv_idx(u8 index, bool is_memset)
308{
309 u8 val = is_memset ? PATTERN_MEMSET_IDX : index;
310
311 return ~val & PATTERN_COUNT_MASK;
312}
313
314static inline u8 gen_src_value(u8 index, bool is_memset)
315{
316 return PATTERN_SRC | gen_inv_idx(index, is_memset);
317}
318
319static inline u8 gen_dst_value(u8 index, bool is_memset)
320{
321 return PATTERN_DST | gen_inv_idx(index, is_memset);
322}
323
324static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len,
325 unsigned int buf_size, bool is_memset)
326{
327 unsigned int i;
328 u8 *buf;
329
330 for (; (buf = *bufs); bufs++) {
331 for (i = 0; i < start; i++)
332 buf[i] = gen_src_value(i, is_memset);
333 for ( ; i < start + len; i++)
334 buf[i] = gen_src_value(i, is_memset) | PATTERN_COPY;
335 for ( ; i < buf_size; i++)
336 buf[i] = gen_src_value(i, is_memset);
337 buf++;
338 }
339}
340
341static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,
342 unsigned int buf_size, bool is_memset)
343{
344 unsigned int i;
345 u8 *buf;
346
347 for (; (buf = *bufs); bufs++) {
348 for (i = 0; i < start; i++)
349 buf[i] = gen_dst_value(i, is_memset);
350 for ( ; i < start + len; i++)
351 buf[i] = gen_dst_value(i, is_memset) |
352 PATTERN_OVERWRITE;
353 for ( ; i < buf_size; i++)
354 buf[i] = gen_dst_value(i, is_memset);
355 }
356}
357
358static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,
359 unsigned int counter, bool is_srcbuf, bool is_memset)
360{
361 u8 diff = actual ^ pattern;
362 u8 expected = pattern | gen_inv_idx(counter, is_memset);
363 const char *thread_name = current->comm;
364
365 if (is_srcbuf)
366 pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n",
367 thread_name, index, expected, actual);
368 else if ((pattern & PATTERN_COPY)
369 && (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
370 pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n",
371 thread_name, index, expected, actual);
372 else if (diff & PATTERN_SRC)
373 pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n",
374 thread_name, index, expected, actual);
375 else
376 pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n",
377 thread_name, index, expected, actual);
378}
379
380static unsigned int dmatest_verify(u8 **bufs, unsigned int start,
381 unsigned int end, unsigned int counter, u8 pattern,
382 bool is_srcbuf, bool is_memset)
383{
384 unsigned int i;
385 unsigned int error_count = 0;
386 u8 actual;
387 u8 expected;
388 u8 *buf;
389 unsigned int counter_orig = counter;
390
391 for (; (buf = *bufs); bufs++) {
392 counter = counter_orig;
393 for (i = start; i < end; i++) {
394 actual = buf[i];
395 expected = pattern | gen_inv_idx(counter, is_memset);
396 if (actual != expected) {
397 if (error_count < MAX_ERROR_COUNT)
398 dmatest_mismatch(actual, pattern, i,
399 counter, is_srcbuf,
400 is_memset);
401 error_count++;
402 }
403 counter++;
404 }
405 }
406
407 if (error_count > MAX_ERROR_COUNT)
408 pr_warn("%s: %u errors suppressed\n",
409 current->comm, error_count - MAX_ERROR_COUNT);
410
411 return error_count;
412}
413
414
415static void dmatest_callback(void *arg)
416{
417 struct dmatest_done *done = arg;
418 struct dmatest_thread *thread =
419 container_of(done, struct dmatest_thread, test_done);
420 if (!thread->done) {
421 done->done = true;
422 wake_up_all(done->wait);
423 } else {
424 /*
425 * If thread->done, it means that this callback occurred
426 * after the parent thread has cleaned up. This can
427 * happen in the case that driver doesn't implement
428 * the terminate_all() functionality and a dma operation
429 * did not occur within the timeout period
430 */
431 WARN(1, "dmatest: Kernel memory may be corrupted!!\n");
432 }
433}
434
435static unsigned int min_odd(unsigned int x, unsigned int y)
436{
437 unsigned int val = min(x, y);
438
439 return val % 2 ? val : val - 1;
440}
441
442static void result(const char *err, unsigned int n, unsigned int src_off,
443 unsigned int dst_off, unsigned int len, unsigned long data)
444{
445 pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
446 current->comm, n, err, src_off, dst_off, len, data);
447}
448
449static void dbg_result(const char *err, unsigned int n, unsigned int src_off,
450 unsigned int dst_off, unsigned int len,
451 unsigned long data)
452{
453 pr_debug("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
454 current->comm, n, err, src_off, dst_off, len, data);
455}
456
457#define verbose_result(err, n, src_off, dst_off, len, data) ({ \
458 if (verbose) \
459 result(err, n, src_off, dst_off, len, data); \
460 else \
461 dbg_result(err, n, src_off, dst_off, len, data);\
462})
463
464static unsigned long long dmatest_persec(s64 runtime, unsigned int val)
465{
466 unsigned long long per_sec = 1000000;
467
468 if (runtime <= 0)
469 return 0;
470
471 /* drop precision until runtime is 32-bits */
472 while (runtime > UINT_MAX) {
473 runtime >>= 1;
474 per_sec <<= 1;
475 }
476
477 per_sec *= val;
478 per_sec = INT_TO_FIXPT(per_sec);
479 do_div(per_sec, runtime);
480
481 return per_sec;
482}
483
484static unsigned long long dmatest_KBs(s64 runtime, unsigned long long len)
485{
486 return FIXPT_TO_INT(dmatest_persec(runtime, len >> 10));
487}
488
489static void __dmatest_free_test_data(struct dmatest_data *d, unsigned int cnt)
490{
491 unsigned int i;
492
493 for (i = 0; i < cnt; i++)
494 kfree(d->raw[i]);
495
496 kfree(d->aligned);
497 kfree(d->raw);
498}
499
500static void dmatest_free_test_data(struct dmatest_data *d)
501{
502 __dmatest_free_test_data(d, d->cnt);
503}
504
505static int dmatest_alloc_test_data(struct dmatest_data *d,
506 unsigned int buf_size, u8 align)
507{
508 unsigned int i = 0;
509
510 d->raw = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL);
511 if (!d->raw)
512 return -ENOMEM;
513
514 d->aligned = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL);
515 if (!d->aligned)
516 goto err;
517
518 for (i = 0; i < d->cnt; i++) {
519 d->raw[i] = kmalloc(buf_size + align, GFP_KERNEL);
520 if (!d->raw[i])
521 goto err;
522
523 /* align to alignment restriction */
524 if (align)
525 d->aligned[i] = PTR_ALIGN(d->raw[i], align);
526 else
527 d->aligned[i] = d->raw[i];
528 }
529
530 return 0;
531err:
532 __dmatest_free_test_data(d, i);
533 return -ENOMEM;
534}
535
536/*
537 * This function repeatedly tests DMA transfers of various lengths and
538 * offsets for a given operation type until it is told to exit by
539 * kthread_stop(). There may be multiple threads running this function
540 * in parallel for a single channel, and there may be multiple channels
541 * being tested in parallel.
542 *
543 * Before each test, the source and destination buffer is initialized
544 * with a known pattern. This pattern is different depending on
545 * whether it's in an area which is supposed to be copied or
546 * overwritten, and different in the source and destination buffers.
547 * So if the DMA engine doesn't copy exactly what we tell it to copy,
548 * we'll notice.
549 */
550static int dmatest_func(void *data)
551{
552 struct dmatest_thread *thread = data;
553 struct dmatest_done *done = &thread->test_done;
554 struct dmatest_info *info;
555 struct dmatest_params *params;
556 struct dma_chan *chan;
557 struct dma_device *dev;
558 unsigned int error_count;
559 unsigned int failed_tests = 0;
560 unsigned int total_tests = 0;
561 dma_cookie_t cookie;
562 enum dma_status status;
563 enum dma_ctrl_flags flags;
564 u8 *pq_coefs = NULL;
565 int ret;
566 unsigned int buf_size;
567 struct dmatest_data *src;
568 struct dmatest_data *dst;
569 int i;
570 ktime_t ktime, start, diff;
571 ktime_t filltime = 0;
572 ktime_t comparetime = 0;
573 s64 runtime = 0;
574 unsigned long long total_len = 0;
575 unsigned long long iops = 0;
576 u8 align = 0;
577 bool is_memset = false;
578 dma_addr_t *srcs;
579 dma_addr_t *dma_pq;
580
581 set_freezable();
582
583 ret = -ENOMEM;
584
585 smp_rmb();
586 thread->pending = false;
587 info = thread->info;
588 params = &info->params;
589 chan = thread->chan;
590 dev = chan->device;
591 src = &thread->src;
592 dst = &thread->dst;
593 if (thread->type == DMA_MEMCPY) {
594 align = params->alignment < 0 ? dev->copy_align :
595 params->alignment;
596 src->cnt = dst->cnt = 1;
597 } else if (thread->type == DMA_MEMSET) {
598 align = params->alignment < 0 ? dev->fill_align :
599 params->alignment;
600 src->cnt = dst->cnt = 1;
601 is_memset = true;
602 } else if (thread->type == DMA_XOR) {
603 /* force odd to ensure dst = src */
604 src->cnt = min_odd(params->xor_sources | 1, dev->max_xor);
605 dst->cnt = 1;
606 align = params->alignment < 0 ? dev->xor_align :
607 params->alignment;
608 } else if (thread->type == DMA_PQ) {
609 /* force odd to ensure dst = src */
610 src->cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0));
611 dst->cnt = 2;
612 align = params->alignment < 0 ? dev->pq_align :
613 params->alignment;
614
615 pq_coefs = kmalloc(params->pq_sources + 1, GFP_KERNEL);
616 if (!pq_coefs)
617 goto err_thread_type;
618
619 for (i = 0; i < src->cnt; i++)
620 pq_coefs[i] = 1;
621 } else
622 goto err_thread_type;
623
624 /* Check if buffer count fits into map count variable (u8) */
625 if ((src->cnt + dst->cnt) >= 255) {
626 pr_err("too many buffers (%d of 255 supported)\n",
627 src->cnt + dst->cnt);
628 goto err_free_coefs;
629 }
630
631 buf_size = params->buf_size;
632 if (1 << align > buf_size) {
633 pr_err("%u-byte buffer too small for %d-byte alignment\n",
634 buf_size, 1 << align);
635 goto err_free_coefs;
636 }
637
638 if (dmatest_alloc_test_data(src, buf_size, align) < 0)
639 goto err_free_coefs;
640
641 if (dmatest_alloc_test_data(dst, buf_size, align) < 0)
642 goto err_src;
643
644 set_user_nice(current, 10);
645
646 srcs = kcalloc(src->cnt, sizeof(dma_addr_t), GFP_KERNEL);
647 if (!srcs)
648 goto err_dst;
649
650 dma_pq = kcalloc(dst->cnt, sizeof(dma_addr_t), GFP_KERNEL);
651 if (!dma_pq)
652 goto err_srcs_array;
653
654 /*
655 * src and dst buffers are freed by ourselves below
656 */
657 flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
658
659 ktime = ktime_get();
660 while (!kthread_should_stop()
661 && !(params->iterations && total_tests >= params->iterations)) {
662 struct dma_async_tx_descriptor *tx = NULL;
663 struct dmaengine_unmap_data *um;
664 dma_addr_t *dsts;
665 unsigned int len;
666
667 total_tests++;
668
669 if (params->transfer_size) {
670 if (params->transfer_size >= buf_size) {
671 pr_err("%u-byte transfer size must be lower than %u-buffer size\n",
672 params->transfer_size, buf_size);
673 break;
674 }
675 len = params->transfer_size;
676 } else if (params->norandom) {
677 len = buf_size;
678 } else {
679 len = dmatest_random() % buf_size + 1;
680 }
681
682 /* Do not alter transfer size explicitly defined by user */
683 if (!params->transfer_size) {
684 len = (len >> align) << align;
685 if (!len)
686 len = 1 << align;
687 }
688 total_len += len;
689
690 if (params->norandom) {
691 src->off = 0;
692 dst->off = 0;
693 } else {
694 src->off = dmatest_random() % (buf_size - len + 1);
695 dst->off = dmatest_random() % (buf_size - len + 1);
696
697 src->off = (src->off >> align) << align;
698 dst->off = (dst->off >> align) << align;
699 }
700
701 if (!params->noverify) {
702 start = ktime_get();
703 dmatest_init_srcs(src->aligned, src->off, len,
704 buf_size, is_memset);
705 dmatest_init_dsts(dst->aligned, dst->off, len,
706 buf_size, is_memset);
707
708 diff = ktime_sub(ktime_get(), start);
709 filltime = ktime_add(filltime, diff);
710 }
711
712 um = dmaengine_get_unmap_data(dev->dev, src->cnt + dst->cnt,
713 GFP_KERNEL);
714 if (!um) {
715 failed_tests++;
716 result("unmap data NULL", total_tests,
717 src->off, dst->off, len, ret);
718 continue;
719 }
720
721 um->len = buf_size;
722 for (i = 0; i < src->cnt; i++) {
723 void *buf = src->aligned[i];
724 struct page *pg = virt_to_page(buf);
725 unsigned long pg_off = offset_in_page(buf);
726
727 um->addr[i] = dma_map_page(dev->dev, pg, pg_off,
728 um->len, DMA_TO_DEVICE);
729 srcs[i] = um->addr[i] + src->off;
730 ret = dma_mapping_error(dev->dev, um->addr[i]);
731 if (ret) {
732 result("src mapping error", total_tests,
733 src->off, dst->off, len, ret);
734 goto error_unmap_continue;
735 }
736 um->to_cnt++;
737 }
738 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
739 dsts = &um->addr[src->cnt];
740 for (i = 0; i < dst->cnt; i++) {
741 void *buf = dst->aligned[i];
742 struct page *pg = virt_to_page(buf);
743 unsigned long pg_off = offset_in_page(buf);
744
745 dsts[i] = dma_map_page(dev->dev, pg, pg_off, um->len,
746 DMA_BIDIRECTIONAL);
747 ret = dma_mapping_error(dev->dev, dsts[i]);
748 if (ret) {
749 result("dst mapping error", total_tests,
750 src->off, dst->off, len, ret);
751 goto error_unmap_continue;
752 }
753 um->bidi_cnt++;
754 }
755
756 if (thread->type == DMA_MEMCPY)
757 tx = dev->device_prep_dma_memcpy(chan,
758 dsts[0] + dst->off,
759 srcs[0], len, flags);
760 else if (thread->type == DMA_MEMSET)
761 tx = dev->device_prep_dma_memset(chan,
762 dsts[0] + dst->off,
763 *(src->aligned[0] + src->off),
764 len, flags);
765 else if (thread->type == DMA_XOR)
766 tx = dev->device_prep_dma_xor(chan,
767 dsts[0] + dst->off,
768 srcs, src->cnt,
769 len, flags);
770 else if (thread->type == DMA_PQ) {
771 for (i = 0; i < dst->cnt; i++)
772 dma_pq[i] = dsts[i] + dst->off;
773 tx = dev->device_prep_dma_pq(chan, dma_pq, srcs,
774 src->cnt, pq_coefs,
775 len, flags);
776 }
777
778 if (!tx) {
779 result("prep error", total_tests, src->off,
780 dst->off, len, ret);
781 msleep(100);
782 goto error_unmap_continue;
783 }
784
785 done->done = false;
786 tx->callback = dmatest_callback;
787 tx->callback_param = done;
788 cookie = tx->tx_submit(tx);
789
790 if (dma_submit_error(cookie)) {
791 result("submit error", total_tests, src->off,
792 dst->off, len, ret);
793 msleep(100);
794 goto error_unmap_continue;
795 }
796 dma_async_issue_pending(chan);
797
798 wait_event_freezable_timeout(thread->done_wait, done->done,
799 msecs_to_jiffies(params->timeout));
800
801 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
802
803 if (!done->done) {
804 result("test timed out", total_tests, src->off, dst->off,
805 len, 0);
806 goto error_unmap_continue;
807 } else if (status != DMA_COMPLETE) {
808 result(status == DMA_ERROR ?
809 "completion error status" :
810 "completion busy status", total_tests, src->off,
811 dst->off, len, ret);
812 goto error_unmap_continue;
813 }
814
815 dmaengine_unmap_put(um);
816
817 if (params->noverify) {
818 verbose_result("test passed", total_tests, src->off,
819 dst->off, len, 0);
820 continue;
821 }
822
823 start = ktime_get();
824 pr_debug("%s: verifying source buffer...\n", current->comm);
825 error_count = dmatest_verify(src->aligned, 0, src->off,
826 0, PATTERN_SRC, true, is_memset);
827 error_count += dmatest_verify(src->aligned, src->off,
828 src->off + len, src->off,
829 PATTERN_SRC | PATTERN_COPY, true, is_memset);
830 error_count += dmatest_verify(src->aligned, src->off + len,
831 buf_size, src->off + len,
832 PATTERN_SRC, true, is_memset);
833
834 pr_debug("%s: verifying dest buffer...\n", current->comm);
835 error_count += dmatest_verify(dst->aligned, 0, dst->off,
836 0, PATTERN_DST, false, is_memset);
837
838 error_count += dmatest_verify(dst->aligned, dst->off,
839 dst->off + len, src->off,
840 PATTERN_SRC | PATTERN_COPY, false, is_memset);
841
842 error_count += dmatest_verify(dst->aligned, dst->off + len,
843 buf_size, dst->off + len,
844 PATTERN_DST, false, is_memset);
845
846 diff = ktime_sub(ktime_get(), start);
847 comparetime = ktime_add(comparetime, diff);
848
849 if (error_count) {
850 result("data error", total_tests, src->off, dst->off,
851 len, error_count);
852 failed_tests++;
853 } else {
854 verbose_result("test passed", total_tests, src->off,
855 dst->off, len, 0);
856 }
857
858 continue;
859
860error_unmap_continue:
861 dmaengine_unmap_put(um);
862 failed_tests++;
863 }
864 ktime = ktime_sub(ktime_get(), ktime);
865 ktime = ktime_sub(ktime, comparetime);
866 ktime = ktime_sub(ktime, filltime);
867 runtime = ktime_to_us(ktime);
868
869 ret = 0;
870 kfree(dma_pq);
871err_srcs_array:
872 kfree(srcs);
873err_dst:
874 dmatest_free_test_data(dst);
875err_src:
876 dmatest_free_test_data(src);
877err_free_coefs:
878 kfree(pq_coefs);
879err_thread_type:
880 iops = dmatest_persec(runtime, total_tests);
881 pr_info("%s: summary %u tests, %u failures %llu.%02llu iops %llu KB/s (%d)\n",
882 current->comm, total_tests, failed_tests,
883 FIXPT_TO_INT(iops), FIXPT_GET_FRAC(iops),
884 dmatest_KBs(runtime, total_len), ret);
885
886 /* terminate all transfers on specified channels */
887 if (ret || failed_tests)
888 dmaengine_terminate_sync(chan);
889
890 thread->done = true;
891 wake_up(&thread_wait);
892
893 return ret;
894}
895
896static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
897{
898 struct dmatest_thread *thread;
899 struct dmatest_thread *_thread;
900 int ret;
901
902 list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
903 ret = kthread_stop(thread->task);
904 pr_debug("thread %s exited with status %d\n",
905 thread->task->comm, ret);
906 list_del(&thread->node);
907 put_task_struct(thread->task);
908 kfree(thread);
909 }
910
911 /* terminate all transfers on specified channels */
912 dmaengine_terminate_sync(dtc->chan);
913
914 kfree(dtc);
915}
916
917static int dmatest_add_threads(struct dmatest_info *info,
918 struct dmatest_chan *dtc, enum dma_transaction_type type)
919{
920 struct dmatest_params *params = &info->params;
921 struct dmatest_thread *thread;
922 struct dma_chan *chan = dtc->chan;
923 char *op;
924 unsigned int i;
925
926 if (type == DMA_MEMCPY)
927 op = "copy";
928 else if (type == DMA_MEMSET)
929 op = "set";
930 else if (type == DMA_XOR)
931 op = "xor";
932 else if (type == DMA_PQ)
933 op = "pq";
934 else
935 return -EINVAL;
936
937 for (i = 0; i < params->threads_per_chan; i++) {
938 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
939 if (!thread) {
940 pr_warn("No memory for %s-%s%u\n",
941 dma_chan_name(chan), op, i);
942 break;
943 }
944 thread->info = info;
945 thread->chan = dtc->chan;
946 thread->type = type;
947 thread->test_done.wait = &thread->done_wait;
948 init_waitqueue_head(&thread->done_wait);
949 smp_wmb();
950 thread->task = kthread_create(dmatest_func, thread, "%s-%s%u",
951 dma_chan_name(chan), op, i);
952 if (IS_ERR(thread->task)) {
953 pr_warn("Failed to create thread %s-%s%u\n",
954 dma_chan_name(chan), op, i);
955 kfree(thread);
956 break;
957 }
958
959 /* srcbuf and dstbuf are allocated by the thread itself */
960 get_task_struct(thread->task);
961 list_add_tail(&thread->node, &dtc->threads);
962 thread->pending = true;
963 }
964
965 return i;
966}
967
968static int dmatest_add_channel(struct dmatest_info *info,
969 struct dma_chan *chan)
970{
971 struct dmatest_chan *dtc;
972 struct dma_device *dma_dev = chan->device;
973 unsigned int thread_count = 0;
974 int cnt;
975
976 dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
977 if (!dtc) {
978 pr_warn("No memory for %s\n", dma_chan_name(chan));
979 return -ENOMEM;
980 }
981
982 dtc->chan = chan;
983 INIT_LIST_HEAD(&dtc->threads);
984
985 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
986 if (dmatest == 0) {
987 cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY);
988 thread_count += cnt > 0 ? cnt : 0;
989 }
990 }
991
992 if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) {
993 if (dmatest == 1) {
994 cnt = dmatest_add_threads(info, dtc, DMA_MEMSET);
995 thread_count += cnt > 0 ? cnt : 0;
996 }
997 }
998
999 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1000 cnt = dmatest_add_threads(info, dtc, DMA_XOR);
1001 thread_count += cnt > 0 ? cnt : 0;
1002 }
1003 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
1004 cnt = dmatest_add_threads(info, dtc, DMA_PQ);
1005 thread_count += cnt > 0 ? cnt : 0;
1006 }
1007
1008 pr_info("Added %u threads using %s\n",
1009 thread_count, dma_chan_name(chan));
1010
1011 list_add_tail(&dtc->node, &info->channels);
1012 info->nr_channels++;
1013
1014 return 0;
1015}
1016
1017static bool filter(struct dma_chan *chan, void *param)
1018{
1019 struct dmatest_params *params = param;
1020
1021 if (!dmatest_match_channel(params, chan) ||
1022 !dmatest_match_device(params, chan->device))
1023 return false;
1024 else
1025 return true;
1026}
1027
1028static void request_channels(struct dmatest_info *info,
1029 enum dma_transaction_type type)
1030{
1031 dma_cap_mask_t mask;
1032
1033 dma_cap_zero(mask);
1034 dma_cap_set(type, mask);
1035 for (;;) {
1036 struct dmatest_params *params = &info->params;
1037 struct dma_chan *chan;
1038
1039 chan = dma_request_channel(mask, filter, params);
1040 if (chan) {
1041 if (dmatest_add_channel(info, chan)) {
1042 dma_release_channel(chan);
1043 break; /* add_channel failed, punt */
1044 }
1045 } else
1046 break; /* no more channels available */
1047 if (params->max_channels &&
1048 info->nr_channels >= params->max_channels)
1049 break; /* we have all we need */
1050 }
1051}
1052
1053static void add_threaded_test(struct dmatest_info *info)
1054{
1055 struct dmatest_params *params = &info->params;
1056
1057 /* Copy test parameters */
1058 params->buf_size = test_buf_size;
1059 strlcpy(params->channel, strim(test_channel), sizeof(params->channel));
1060 strlcpy(params->device, strim(test_device), sizeof(params->device));
1061 params->threads_per_chan = threads_per_chan;
1062 params->max_channels = max_channels;
1063 params->iterations = iterations;
1064 params->xor_sources = xor_sources;
1065 params->pq_sources = pq_sources;
1066 params->timeout = timeout;
1067 params->noverify = noverify;
1068 params->norandom = norandom;
1069 params->alignment = alignment;
1070 params->transfer_size = transfer_size;
1071
1072 request_channels(info, DMA_MEMCPY);
1073 request_channels(info, DMA_MEMSET);
1074 request_channels(info, DMA_XOR);
1075 request_channels(info, DMA_PQ);
1076}
1077
1078static void run_pending_tests(struct dmatest_info *info)
1079{
1080 struct dmatest_chan *dtc;
1081 unsigned int thread_count = 0;
1082
1083 list_for_each_entry(dtc, &info->channels, node) {
1084 struct dmatest_thread *thread;
1085
1086 thread_count = 0;
1087 list_for_each_entry(thread, &dtc->threads, node) {
1088 wake_up_process(thread->task);
1089 thread_count++;
1090 }
1091 pr_info("Started %u threads using %s\n",
1092 thread_count, dma_chan_name(dtc->chan));
1093 }
1094}
1095
1096static void stop_threaded_test(struct dmatest_info *info)
1097{
1098 struct dmatest_chan *dtc, *_dtc;
1099 struct dma_chan *chan;
1100
1101 list_for_each_entry_safe(dtc, _dtc, &info->channels, node) {
1102 list_del(&dtc->node);
1103 chan = dtc->chan;
1104 dmatest_cleanup_channel(dtc);
1105 pr_debug("dropped channel %s\n", dma_chan_name(chan));
1106 dma_release_channel(chan);
1107 }
1108
1109 info->nr_channels = 0;
1110}
1111
1112static void start_threaded_tests(struct dmatest_info *info)
1113{
1114 /* we might be called early to set run=, defer running until all
1115 * parameters have been evaluated
1116 */
1117 if (!info->did_init)
1118 return;
1119
1120 run_pending_tests(info);
1121}
1122
1123static int dmatest_run_get(char *val, const struct kernel_param *kp)
1124{
1125 struct dmatest_info *info = &test_info;
1126
1127 mutex_lock(&info->lock);
1128 if (is_threaded_test_run(info)) {
1129 dmatest_run = true;
1130 } else {
1131 if (!is_threaded_test_pending(info))
1132 stop_threaded_test(info);
1133 dmatest_run = false;
1134 }
1135 mutex_unlock(&info->lock);
1136
1137 return param_get_bool(val, kp);
1138}
1139
1140static int dmatest_run_set(const char *val, const struct kernel_param *kp)
1141{
1142 struct dmatest_info *info = &test_info;
1143 int ret;
1144
1145 mutex_lock(&info->lock);
1146 ret = param_set_bool(val, kp);
1147 if (ret) {
1148 mutex_unlock(&info->lock);
1149 return ret;
1150 } else if (dmatest_run) {
1151 if (is_threaded_test_pending(info))
1152 start_threaded_tests(info);
1153 else
1154 pr_info("Could not start test, no channels configured\n");
1155 } else {
1156 stop_threaded_test(info);
1157 }
1158
1159 mutex_unlock(&info->lock);
1160
1161 return ret;
1162}
1163
1164static int dmatest_chan_set(const char *val, const struct kernel_param *kp)
1165{
1166 struct dmatest_info *info = &test_info;
1167 struct dmatest_chan *dtc;
1168 char chan_reset_val[20];
1169 int ret = 0;
1170
1171 mutex_lock(&info->lock);
1172 ret = param_set_copystring(val, kp);
1173 if (ret) {
1174 mutex_unlock(&info->lock);
1175 return ret;
1176 }
1177 /*Clear any previously run threads */
1178 if (!is_threaded_test_run(info) && !is_threaded_test_pending(info))
1179 stop_threaded_test(info);
1180 /* Reject channels that are already registered */
1181 if (is_threaded_test_pending(info)) {
1182 list_for_each_entry(dtc, &info->channels, node) {
1183 if (strcmp(dma_chan_name(dtc->chan),
1184 strim(test_channel)) == 0) {
1185 dtc = list_last_entry(&info->channels,
1186 struct dmatest_chan,
1187 node);
1188 strlcpy(chan_reset_val,
1189 dma_chan_name(dtc->chan),
1190 sizeof(chan_reset_val));
1191 ret = -EBUSY;
1192 goto add_chan_err;
1193 }
1194 }
1195 }
1196
1197 add_threaded_test(info);
1198
1199 /* Check if channel was added successfully */
1200 dtc = list_last_entry(&info->channels, struct dmatest_chan, node);
1201
1202 if (dtc->chan) {
1203 /*
1204 * if new channel was not successfully added, revert the
1205 * "test_channel" string to the name of the last successfully
1206 * added channel. exception for when users issues empty string
1207 * to channel parameter.
1208 */
1209 if ((strcmp(dma_chan_name(dtc->chan), strim(test_channel)) != 0)
1210 && (strcmp("", strim(test_channel)) != 0)) {
1211 ret = -EINVAL;
1212 strlcpy(chan_reset_val, dma_chan_name(dtc->chan),
1213 sizeof(chan_reset_val));
1214 goto add_chan_err;
1215 }
1216
1217 } else {
1218 /* Clear test_channel if no channels were added successfully */
1219 strlcpy(chan_reset_val, "", sizeof(chan_reset_val));
1220 ret = -EBUSY;
1221 goto add_chan_err;
1222 }
1223
1224 mutex_unlock(&info->lock);
1225
1226 return ret;
1227
1228add_chan_err:
1229 param_set_copystring(chan_reset_val, kp);
1230 mutex_unlock(&info->lock);
1231
1232 return ret;
1233}
1234
1235static int dmatest_chan_get(char *val, const struct kernel_param *kp)
1236{
1237 struct dmatest_info *info = &test_info;
1238
1239 mutex_lock(&info->lock);
1240 if (!is_threaded_test_run(info) && !is_threaded_test_pending(info)) {
1241 stop_threaded_test(info);
1242 strlcpy(test_channel, "", sizeof(test_channel));
1243 }
1244 mutex_unlock(&info->lock);
1245
1246 return param_get_string(val, kp);
1247}
1248
1249static int dmatest_test_list_get(char *val, const struct kernel_param *kp)
1250{
1251 struct dmatest_info *info = &test_info;
1252 struct dmatest_chan *dtc;
1253 unsigned int thread_count = 0;
1254
1255 list_for_each_entry(dtc, &info->channels, node) {
1256 struct dmatest_thread *thread;
1257
1258 thread_count = 0;
1259 list_for_each_entry(thread, &dtc->threads, node) {
1260 thread_count++;
1261 }
1262 pr_info("%u threads using %s\n",
1263 thread_count, dma_chan_name(dtc->chan));
1264 }
1265
1266 return 0;
1267}
1268
1269static int __init dmatest_init(void)
1270{
1271 struct dmatest_info *info = &test_info;
1272 struct dmatest_params *params = &info->params;
1273
1274 if (dmatest_run) {
1275 mutex_lock(&info->lock);
1276 add_threaded_test(info);
1277 run_pending_tests(info);
1278 mutex_unlock(&info->lock);
1279 }
1280
1281 if (params->iterations && wait)
1282 wait_event(thread_wait, !is_threaded_test_run(info));
1283
1284 /* module parameters are stable, inittime tests are started,
1285 * let userspace take over 'run' control
1286 */
1287 info->did_init = true;
1288
1289 return 0;
1290}
1291/* when compiled-in wait for drivers to load first */
1292late_initcall(dmatest_init);
1293
1294static void __exit dmatest_exit(void)
1295{
1296 struct dmatest_info *info = &test_info;
1297
1298 mutex_lock(&info->lock);
1299 stop_threaded_test(info);
1300 mutex_unlock(&info->lock);
1301}
1302module_exit(dmatest_exit);
1303
1304MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1305MODULE_LICENSE("GPL v2");
1306