1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (C) 2008 Advanced Micro Devices, Inc. |
4 | * |
5 | * Author: Joerg Roedel <joerg.roedel@amd.com> |
6 | */ |
7 | |
8 | #define pr_fmt(fmt) "DMA-API: " fmt |
9 | |
10 | #include <linux/sched/task_stack.h> |
11 | #include <linux/scatterlist.h> |
12 | #include <linux/dma-map-ops.h> |
13 | #include <linux/sched/task.h> |
14 | #include <linux/stacktrace.h> |
15 | #include <linux/spinlock.h> |
16 | #include <linux/vmalloc.h> |
17 | #include <linux/debugfs.h> |
18 | #include <linux/uaccess.h> |
19 | #include <linux/export.h> |
20 | #include <linux/device.h> |
21 | #include <linux/types.h> |
22 | #include <linux/sched.h> |
23 | #include <linux/ctype.h> |
24 | #include <linux/list.h> |
25 | #include <linux/slab.h> |
26 | #include <asm/sections.h> |
27 | #include "debug.h" |
28 | |
29 | #define HASH_SIZE 16384ULL |
30 | #define HASH_FN_SHIFT 13 |
31 | #define HASH_FN_MASK (HASH_SIZE - 1) |
32 | |
33 | #define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16) |
34 | /* If the pool runs out, add this many new entries at once */ |
35 | #define DMA_DEBUG_DYNAMIC_ENTRIES (PAGE_SIZE / sizeof(struct dma_debug_entry)) |
36 | |
37 | enum { |
38 | dma_debug_single, |
39 | dma_debug_sg, |
40 | dma_debug_coherent, |
41 | dma_debug_resource, |
42 | }; |
43 | |
44 | enum map_err_types { |
45 | MAP_ERR_CHECK_NOT_APPLICABLE, |
46 | MAP_ERR_NOT_CHECKED, |
47 | MAP_ERR_CHECKED, |
48 | }; |
49 | |
50 | #define DMA_DEBUG_STACKTRACE_ENTRIES 5 |
51 | |
52 | /** |
53 | * struct dma_debug_entry - track a dma_map* or dma_alloc_coherent mapping |
54 | * @list: node on pre-allocated free_entries list |
55 | * @dev: 'dev' argument to dma_map_{page|single|sg} or dma_alloc_coherent |
56 | * @dev_addr: dma address |
57 | * @size: length of the mapping |
58 | * @type: single, page, sg, coherent |
59 | * @direction: enum dma_data_direction |
60 | * @sg_call_ents: 'nents' from dma_map_sg |
61 | * @sg_mapped_ents: 'mapped_ents' from dma_map_sg |
62 | * @pfn: page frame of the start address |
63 | * @offset: offset of mapping relative to pfn |
64 | * @map_err_type: track whether dma_mapping_error() was checked |
65 | * @stacktrace: support backtraces when a violation is detected |
66 | */ |
67 | struct dma_debug_entry { |
68 | struct list_head list; |
69 | struct device *dev; |
70 | u64 dev_addr; |
71 | u64 size; |
72 | int type; |
73 | int direction; |
74 | int sg_call_ents; |
75 | int sg_mapped_ents; |
76 | unsigned long pfn; |
77 | size_t offset; |
78 | enum map_err_types map_err_type; |
79 | #ifdef CONFIG_STACKTRACE |
80 | unsigned int stack_len; |
81 | unsigned long stack_entries[DMA_DEBUG_STACKTRACE_ENTRIES]; |
82 | #endif |
83 | } ____cacheline_aligned_in_smp; |
84 | |
85 | typedef bool (*match_fn)(struct dma_debug_entry *, struct dma_debug_entry *); |
86 | |
87 | struct hash_bucket { |
88 | struct list_head list; |
89 | spinlock_t lock; |
90 | }; |
91 | |
92 | /* Hash list to save the allocated dma addresses */ |
93 | static struct hash_bucket dma_entry_hash[HASH_SIZE]; |
94 | /* List of pre-allocated dma_debug_entry's */ |
95 | static LIST_HEAD(free_entries); |
96 | /* Lock for the list above */ |
97 | static DEFINE_SPINLOCK(free_entries_lock); |
98 | |
99 | /* Global disable flag - will be set in case of an error */ |
100 | static bool global_disable __read_mostly; |
101 | |
102 | /* Early initialization disable flag, set at the end of dma_debug_init */ |
103 | static bool dma_debug_initialized __read_mostly; |
104 | |
105 | static inline bool dma_debug_disabled(void) |
106 | { |
107 | return global_disable || !dma_debug_initialized; |
108 | } |
109 | |
110 | /* Global error count */ |
111 | static u32 error_count; |
112 | |
113 | /* Global error show enable*/ |
114 | static u32 show_all_errors __read_mostly; |
115 | /* Number of errors to show */ |
116 | static u32 show_num_errors = 1; |
117 | |
118 | static u32 num_free_entries; |
119 | static u32 min_free_entries; |
120 | static u32 nr_total_entries; |
121 | |
122 | /* number of preallocated entries requested by kernel cmdline */ |
123 | static u32 nr_prealloc_entries = PREALLOC_DMA_DEBUG_ENTRIES; |
124 | |
125 | /* per-driver filter related state */ |
126 | |
127 | #define NAME_MAX_LEN 64 |
128 | |
129 | static char current_driver_name[NAME_MAX_LEN] __read_mostly; |
130 | static struct device_driver *current_driver __read_mostly; |
131 | |
132 | static DEFINE_RWLOCK(driver_name_lock); |
133 | |
134 | static const char *const maperr2str[] = { |
135 | [MAP_ERR_CHECK_NOT_APPLICABLE] = "dma map error check not applicable" , |
136 | [MAP_ERR_NOT_CHECKED] = "dma map error not checked" , |
137 | [MAP_ERR_CHECKED] = "dma map error checked" , |
138 | }; |
139 | |
140 | static const char *type2name[] = { |
141 | [dma_debug_single] = "single" , |
142 | [dma_debug_sg] = "scatter-gather" , |
143 | [dma_debug_coherent] = "coherent" , |
144 | [dma_debug_resource] = "resource" , |
145 | }; |
146 | |
147 | static const char *dir2name[] = { |
148 | [DMA_BIDIRECTIONAL] = "DMA_BIDIRECTIONAL" , |
149 | [DMA_TO_DEVICE] = "DMA_TO_DEVICE" , |
150 | [DMA_FROM_DEVICE] = "DMA_FROM_DEVICE" , |
151 | [DMA_NONE] = "DMA_NONE" , |
152 | }; |
153 | |
154 | /* |
155 | * The access to some variables in this macro is racy. We can't use atomic_t |
156 | * here because all these variables are exported to debugfs. Some of them even |
157 | * writeable. This is also the reason why a lock won't help much. But anyway, |
158 | * the races are no big deal. Here is why: |
159 | * |
160 | * error_count: the addition is racy, but the worst thing that can happen is |
161 | * that we don't count some errors |
162 | * show_num_errors: the subtraction is racy. Also no big deal because in |
163 | * worst case this will result in one warning more in the |
164 | * system log than the user configured. This variable is |
165 | * writeable via debugfs. |
166 | */ |
167 | static inline void dump_entry_trace(struct dma_debug_entry *entry) |
168 | { |
169 | #ifdef CONFIG_STACKTRACE |
170 | if (entry) { |
171 | pr_warn("Mapped at:\n" ); |
172 | stack_trace_print(trace: entry->stack_entries, nr_entries: entry->stack_len, spaces: 0); |
173 | } |
174 | #endif |
175 | } |
176 | |
177 | static bool driver_filter(struct device *dev) |
178 | { |
179 | struct device_driver *drv; |
180 | unsigned long flags; |
181 | bool ret; |
182 | |
183 | /* driver filter off */ |
184 | if (likely(!current_driver_name[0])) |
185 | return true; |
186 | |
187 | /* driver filter on and initialized */ |
188 | if (current_driver && dev && dev->driver == current_driver) |
189 | return true; |
190 | |
191 | /* driver filter on, but we can't filter on a NULL device... */ |
192 | if (!dev) |
193 | return false; |
194 | |
195 | if (current_driver || !current_driver_name[0]) |
196 | return false; |
197 | |
198 | /* driver filter on but not yet initialized */ |
199 | drv = dev->driver; |
200 | if (!drv) |
201 | return false; |
202 | |
203 | /* lock to protect against change of current_driver_name */ |
204 | read_lock_irqsave(&driver_name_lock, flags); |
205 | |
206 | ret = false; |
207 | if (drv->name && |
208 | strncmp(current_driver_name, drv->name, NAME_MAX_LEN - 1) == 0) { |
209 | current_driver = drv; |
210 | ret = true; |
211 | } |
212 | |
213 | read_unlock_irqrestore(&driver_name_lock, flags); |
214 | |
215 | return ret; |
216 | } |
217 | |
218 | #define err_printk(dev, entry, format, arg...) do { \ |
219 | error_count += 1; \ |
220 | if (driver_filter(dev) && \ |
221 | (show_all_errors || show_num_errors > 0)) { \ |
222 | WARN(1, pr_fmt("%s %s: ") format, \ |
223 | dev ? dev_driver_string(dev) : "NULL", \ |
224 | dev ? dev_name(dev) : "NULL", ## arg); \ |
225 | dump_entry_trace(entry); \ |
226 | } \ |
227 | if (!show_all_errors && show_num_errors > 0) \ |
228 | show_num_errors -= 1; \ |
229 | } while (0); |
230 | |
231 | /* |
232 | * Hash related functions |
233 | * |
234 | * Every DMA-API request is saved into a struct dma_debug_entry. To |
235 | * have quick access to these structs they are stored into a hash. |
236 | */ |
237 | static int hash_fn(struct dma_debug_entry *entry) |
238 | { |
239 | /* |
240 | * Hash function is based on the dma address. |
241 | * We use bits 20-27 here as the index into the hash |
242 | */ |
243 | return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK; |
244 | } |
245 | |
246 | /* |
247 | * Request exclusive access to a hash bucket for a given dma_debug_entry. |
248 | */ |
249 | static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry, |
250 | unsigned long *flags) |
251 | __acquires(&dma_entry_hash[idx].lock) |
252 | { |
253 | int idx = hash_fn(entry); |
254 | unsigned long __flags; |
255 | |
256 | spin_lock_irqsave(&dma_entry_hash[idx].lock, __flags); |
257 | *flags = __flags; |
258 | return &dma_entry_hash[idx]; |
259 | } |
260 | |
261 | /* |
262 | * Give up exclusive access to the hash bucket |
263 | */ |
264 | static void put_hash_bucket(struct hash_bucket *bucket, |
265 | unsigned long flags) |
266 | __releases(&bucket->lock) |
267 | { |
268 | spin_unlock_irqrestore(lock: &bucket->lock, flags); |
269 | } |
270 | |
271 | static bool exact_match(struct dma_debug_entry *a, struct dma_debug_entry *b) |
272 | { |
273 | return ((a->dev_addr == b->dev_addr) && |
274 | (a->dev == b->dev)) ? true : false; |
275 | } |
276 | |
277 | static bool containing_match(struct dma_debug_entry *a, |
278 | struct dma_debug_entry *b) |
279 | { |
280 | if (a->dev != b->dev) |
281 | return false; |
282 | |
283 | if ((b->dev_addr <= a->dev_addr) && |
284 | ((b->dev_addr + b->size) >= (a->dev_addr + a->size))) |
285 | return true; |
286 | |
287 | return false; |
288 | } |
289 | |
290 | /* |
291 | * Search a given entry in the hash bucket list |
292 | */ |
293 | static struct dma_debug_entry *__hash_bucket_find(struct hash_bucket *bucket, |
294 | struct dma_debug_entry *ref, |
295 | match_fn match) |
296 | { |
297 | struct dma_debug_entry *entry, *ret = NULL; |
298 | int matches = 0, match_lvl, last_lvl = -1; |
299 | |
300 | list_for_each_entry(entry, &bucket->list, list) { |
301 | if (!match(ref, entry)) |
302 | continue; |
303 | |
304 | /* |
305 | * Some drivers map the same physical address multiple |
306 | * times. Without a hardware IOMMU this results in the |
307 | * same device addresses being put into the dma-debug |
308 | * hash multiple times too. This can result in false |
309 | * positives being reported. Therefore we implement a |
310 | * best-fit algorithm here which returns the entry from |
311 | * the hash which fits best to the reference value |
312 | * instead of the first-fit. |
313 | */ |
314 | matches += 1; |
315 | match_lvl = 0; |
316 | entry->size == ref->size ? ++match_lvl : 0; |
317 | entry->type == ref->type ? ++match_lvl : 0; |
318 | entry->direction == ref->direction ? ++match_lvl : 0; |
319 | entry->sg_call_ents == ref->sg_call_ents ? ++match_lvl : 0; |
320 | |
321 | if (match_lvl == 4) { |
322 | /* perfect-fit - return the result */ |
323 | return entry; |
324 | } else if (match_lvl > last_lvl) { |
325 | /* |
326 | * We found an entry that fits better then the |
327 | * previous one or it is the 1st match. |
328 | */ |
329 | last_lvl = match_lvl; |
330 | ret = entry; |
331 | } |
332 | } |
333 | |
334 | /* |
335 | * If we have multiple matches but no perfect-fit, just return |
336 | * NULL. |
337 | */ |
338 | ret = (matches == 1) ? ret : NULL; |
339 | |
340 | return ret; |
341 | } |
342 | |
343 | static struct dma_debug_entry *bucket_find_exact(struct hash_bucket *bucket, |
344 | struct dma_debug_entry *ref) |
345 | { |
346 | return __hash_bucket_find(bucket, ref, match: exact_match); |
347 | } |
348 | |
349 | static struct dma_debug_entry *bucket_find_contain(struct hash_bucket **bucket, |
350 | struct dma_debug_entry *ref, |
351 | unsigned long *flags) |
352 | { |
353 | |
354 | struct dma_debug_entry *entry, index = *ref; |
355 | int limit = min(HASH_SIZE, (index.dev_addr >> HASH_FN_SHIFT) + 1); |
356 | |
357 | for (int i = 0; i < limit; i++) { |
358 | entry = __hash_bucket_find(bucket: *bucket, ref, match: containing_match); |
359 | |
360 | if (entry) |
361 | return entry; |
362 | |
363 | /* |
364 | * Nothing found, go back a hash bucket |
365 | */ |
366 | put_hash_bucket(bucket: *bucket, flags: *flags); |
367 | index.dev_addr -= (1 << HASH_FN_SHIFT); |
368 | *bucket = get_hash_bucket(entry: &index, flags); |
369 | } |
370 | |
371 | return NULL; |
372 | } |
373 | |
374 | /* |
375 | * Add an entry to a hash bucket |
376 | */ |
377 | static void hash_bucket_add(struct hash_bucket *bucket, |
378 | struct dma_debug_entry *entry) |
379 | { |
380 | list_add_tail(new: &entry->list, head: &bucket->list); |
381 | } |
382 | |
383 | /* |
384 | * Remove entry from a hash bucket list |
385 | */ |
386 | static void hash_bucket_del(struct dma_debug_entry *entry) |
387 | { |
388 | list_del(entry: &entry->list); |
389 | } |
390 | |
391 | static unsigned long long phys_addr(struct dma_debug_entry *entry) |
392 | { |
393 | if (entry->type == dma_debug_resource) |
394 | return __pfn_to_phys(entry->pfn) + entry->offset; |
395 | |
396 | return page_to_phys(pfn_to_page(entry->pfn)) + entry->offset; |
397 | } |
398 | |
399 | /* |
400 | * For each mapping (initial cacheline in the case of |
401 | * dma_alloc_coherent/dma_map_page, initial cacheline in each page of a |
402 | * scatterlist, or the cacheline specified in dma_map_single) insert |
403 | * into this tree using the cacheline as the key. At |
404 | * dma_unmap_{single|sg|page} or dma_free_coherent delete the entry. If |
405 | * the entry already exists at insertion time add a tag as a reference |
406 | * count for the overlapping mappings. For now, the overlap tracking |
407 | * just ensures that 'unmaps' balance 'maps' before marking the |
408 | * cacheline idle, but we should also be flagging overlaps as an API |
409 | * violation. |
410 | * |
411 | * Memory usage is mostly constrained by the maximum number of available |
412 | * dma-debug entries in that we need a free dma_debug_entry before |
413 | * inserting into the tree. In the case of dma_map_page and |
414 | * dma_alloc_coherent there is only one dma_debug_entry and one |
415 | * dma_active_cacheline entry to track per event. dma_map_sg(), on the |
416 | * other hand, consumes a single dma_debug_entry, but inserts 'nents' |
417 | * entries into the tree. |
418 | */ |
419 | static RADIX_TREE(dma_active_cacheline, GFP_ATOMIC); |
420 | static DEFINE_SPINLOCK(radix_lock); |
421 | #define ACTIVE_CACHELINE_MAX_OVERLAP ((1 << RADIX_TREE_MAX_TAGS) - 1) |
422 | #define CACHELINE_PER_PAGE_SHIFT (PAGE_SHIFT - L1_CACHE_SHIFT) |
423 | #define CACHELINES_PER_PAGE (1 << CACHELINE_PER_PAGE_SHIFT) |
424 | |
425 | static phys_addr_t to_cacheline_number(struct dma_debug_entry *entry) |
426 | { |
427 | return (entry->pfn << CACHELINE_PER_PAGE_SHIFT) + |
428 | (entry->offset >> L1_CACHE_SHIFT); |
429 | } |
430 | |
431 | static int active_cacheline_read_overlap(phys_addr_t cln) |
432 | { |
433 | int overlap = 0, i; |
434 | |
435 | for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--) |
436 | if (radix_tree_tag_get(&dma_active_cacheline, index: cln, tag: i)) |
437 | overlap |= 1 << i; |
438 | return overlap; |
439 | } |
440 | |
441 | static int active_cacheline_set_overlap(phys_addr_t cln, int overlap) |
442 | { |
443 | int i; |
444 | |
445 | if (overlap > ACTIVE_CACHELINE_MAX_OVERLAP || overlap < 0) |
446 | return overlap; |
447 | |
448 | for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--) |
449 | if (overlap & 1 << i) |
450 | radix_tree_tag_set(&dma_active_cacheline, index: cln, tag: i); |
451 | else |
452 | radix_tree_tag_clear(&dma_active_cacheline, index: cln, tag: i); |
453 | |
454 | return overlap; |
455 | } |
456 | |
457 | static void active_cacheline_inc_overlap(phys_addr_t cln) |
458 | { |
459 | int overlap = active_cacheline_read_overlap(cln); |
460 | |
461 | overlap = active_cacheline_set_overlap(cln, overlap: ++overlap); |
462 | |
463 | /* If we overflowed the overlap counter then we're potentially |
464 | * leaking dma-mappings. |
465 | */ |
466 | WARN_ONCE(overlap > ACTIVE_CACHELINE_MAX_OVERLAP, |
467 | pr_fmt("exceeded %d overlapping mappings of cacheline %pa\n" ), |
468 | ACTIVE_CACHELINE_MAX_OVERLAP, &cln); |
469 | } |
470 | |
471 | static int active_cacheline_dec_overlap(phys_addr_t cln) |
472 | { |
473 | int overlap = active_cacheline_read_overlap(cln); |
474 | |
475 | return active_cacheline_set_overlap(cln, overlap: --overlap); |
476 | } |
477 | |
478 | static int active_cacheline_insert(struct dma_debug_entry *entry) |
479 | { |
480 | phys_addr_t cln = to_cacheline_number(entry); |
481 | unsigned long flags; |
482 | int rc; |
483 | |
484 | /* If the device is not writing memory then we don't have any |
485 | * concerns about the cpu consuming stale data. This mitigates |
486 | * legitimate usages of overlapping mappings. |
487 | */ |
488 | if (entry->direction == DMA_TO_DEVICE) |
489 | return 0; |
490 | |
491 | spin_lock_irqsave(&radix_lock, flags); |
492 | rc = radix_tree_insert(&dma_active_cacheline, index: cln, entry); |
493 | if (rc == -EEXIST) |
494 | active_cacheline_inc_overlap(cln); |
495 | spin_unlock_irqrestore(lock: &radix_lock, flags); |
496 | |
497 | return rc; |
498 | } |
499 | |
500 | static void active_cacheline_remove(struct dma_debug_entry *entry) |
501 | { |
502 | phys_addr_t cln = to_cacheline_number(entry); |
503 | unsigned long flags; |
504 | |
505 | /* ...mirror the insert case */ |
506 | if (entry->direction == DMA_TO_DEVICE) |
507 | return; |
508 | |
509 | spin_lock_irqsave(&radix_lock, flags); |
510 | /* since we are counting overlaps the final put of the |
511 | * cacheline will occur when the overlap count is 0. |
512 | * active_cacheline_dec_overlap() returns -1 in that case |
513 | */ |
514 | if (active_cacheline_dec_overlap(cln) < 0) |
515 | radix_tree_delete(&dma_active_cacheline, cln); |
516 | spin_unlock_irqrestore(lock: &radix_lock, flags); |
517 | } |
518 | |
519 | /* |
520 | * Dump mappings entries on kernel space for debugging purposes |
521 | */ |
522 | void debug_dma_dump_mappings(struct device *dev) |
523 | { |
524 | int idx; |
525 | phys_addr_t cln; |
526 | |
527 | for (idx = 0; idx < HASH_SIZE; idx++) { |
528 | struct hash_bucket *bucket = &dma_entry_hash[idx]; |
529 | struct dma_debug_entry *entry; |
530 | unsigned long flags; |
531 | |
532 | spin_lock_irqsave(&bucket->lock, flags); |
533 | list_for_each_entry(entry, &bucket->list, list) { |
534 | if (!dev || dev == entry->dev) { |
535 | cln = to_cacheline_number(entry); |
536 | dev_info(entry->dev, |
537 | "%s idx %d P=%llx N=%lx D=%llx L=%llx cln=%pa %s %s\n" , |
538 | type2name[entry->type], idx, |
539 | phys_addr(entry), entry->pfn, |
540 | entry->dev_addr, entry->size, |
541 | &cln, dir2name[entry->direction], |
542 | maperr2str[entry->map_err_type]); |
543 | } |
544 | } |
545 | spin_unlock_irqrestore(lock: &bucket->lock, flags); |
546 | |
547 | cond_resched(); |
548 | } |
549 | } |
550 | |
551 | /* |
552 | * Dump mappings entries on user space via debugfs |
553 | */ |
554 | static int dump_show(struct seq_file *seq, void *v) |
555 | { |
556 | int idx; |
557 | phys_addr_t cln; |
558 | |
559 | for (idx = 0; idx < HASH_SIZE; idx++) { |
560 | struct hash_bucket *bucket = &dma_entry_hash[idx]; |
561 | struct dma_debug_entry *entry; |
562 | unsigned long flags; |
563 | |
564 | spin_lock_irqsave(&bucket->lock, flags); |
565 | list_for_each_entry(entry, &bucket->list, list) { |
566 | cln = to_cacheline_number(entry); |
567 | seq_printf(m: seq, |
568 | fmt: "%s %s %s idx %d P=%llx N=%lx D=%llx L=%llx cln=%pa %s %s\n" , |
569 | dev_driver_string(dev: entry->dev), |
570 | dev_name(dev: entry->dev), |
571 | type2name[entry->type], idx, |
572 | phys_addr(entry), entry->pfn, |
573 | entry->dev_addr, entry->size, |
574 | &cln, dir2name[entry->direction], |
575 | maperr2str[entry->map_err_type]); |
576 | } |
577 | spin_unlock_irqrestore(lock: &bucket->lock, flags); |
578 | } |
579 | return 0; |
580 | } |
581 | DEFINE_SHOW_ATTRIBUTE(dump); |
582 | |
583 | /* |
584 | * Wrapper function for adding an entry to the hash. |
585 | * This function takes care of locking itself. |
586 | */ |
587 | static void add_dma_entry(struct dma_debug_entry *entry, unsigned long attrs) |
588 | { |
589 | struct hash_bucket *bucket; |
590 | unsigned long flags; |
591 | int rc; |
592 | |
593 | bucket = get_hash_bucket(entry, flags: &flags); |
594 | hash_bucket_add(bucket, entry); |
595 | put_hash_bucket(bucket, flags); |
596 | |
597 | rc = active_cacheline_insert(entry); |
598 | if (rc == -ENOMEM) { |
599 | pr_err_once("cacheline tracking ENOMEM, dma-debug disabled\n" ); |
600 | global_disable = true; |
601 | } else if (rc == -EEXIST && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) { |
602 | err_printk(entry->dev, entry, |
603 | "cacheline tracking EEXIST, overlapping mappings aren't supported\n" ); |
604 | } |
605 | } |
606 | |
607 | static int dma_debug_create_entries(gfp_t gfp) |
608 | { |
609 | struct dma_debug_entry *entry; |
610 | int i; |
611 | |
612 | entry = (void *)get_zeroed_page(gfp_mask: gfp); |
613 | if (!entry) |
614 | return -ENOMEM; |
615 | |
616 | for (i = 0; i < DMA_DEBUG_DYNAMIC_ENTRIES; i++) |
617 | list_add_tail(new: &entry[i].list, head: &free_entries); |
618 | |
619 | num_free_entries += DMA_DEBUG_DYNAMIC_ENTRIES; |
620 | nr_total_entries += DMA_DEBUG_DYNAMIC_ENTRIES; |
621 | |
622 | return 0; |
623 | } |
624 | |
625 | static struct dma_debug_entry *__dma_entry_alloc(void) |
626 | { |
627 | struct dma_debug_entry *entry; |
628 | |
629 | entry = list_entry(free_entries.next, struct dma_debug_entry, list); |
630 | list_del(entry: &entry->list); |
631 | memset(entry, 0, sizeof(*entry)); |
632 | |
633 | num_free_entries -= 1; |
634 | if (num_free_entries < min_free_entries) |
635 | min_free_entries = num_free_entries; |
636 | |
637 | return entry; |
638 | } |
639 | |
640 | /* |
641 | * This should be called outside of free_entries_lock scope to avoid potential |
642 | * deadlocks with serial consoles that use DMA. |
643 | */ |
644 | static void __dma_entry_alloc_check_leak(u32 nr_entries) |
645 | { |
646 | u32 tmp = nr_entries % nr_prealloc_entries; |
647 | |
648 | /* Shout each time we tick over some multiple of the initial pool */ |
649 | if (tmp < DMA_DEBUG_DYNAMIC_ENTRIES) { |
650 | pr_info("dma_debug_entry pool grown to %u (%u00%%)\n" , |
651 | nr_entries, |
652 | (nr_entries / nr_prealloc_entries)); |
653 | } |
654 | } |
655 | |
656 | /* struct dma_entry allocator |
657 | * |
658 | * The next two functions implement the allocator for |
659 | * struct dma_debug_entries. |
660 | */ |
661 | static struct dma_debug_entry *dma_entry_alloc(void) |
662 | { |
663 | bool alloc_check_leak = false; |
664 | struct dma_debug_entry *entry; |
665 | unsigned long flags; |
666 | u32 nr_entries; |
667 | |
668 | spin_lock_irqsave(&free_entries_lock, flags); |
669 | if (num_free_entries == 0) { |
670 | if (dma_debug_create_entries(GFP_ATOMIC)) { |
671 | global_disable = true; |
672 | spin_unlock_irqrestore(lock: &free_entries_lock, flags); |
673 | pr_err("debugging out of memory - disabling\n" ); |
674 | return NULL; |
675 | } |
676 | alloc_check_leak = true; |
677 | nr_entries = nr_total_entries; |
678 | } |
679 | |
680 | entry = __dma_entry_alloc(); |
681 | |
682 | spin_unlock_irqrestore(lock: &free_entries_lock, flags); |
683 | |
684 | if (alloc_check_leak) |
685 | __dma_entry_alloc_check_leak(nr_entries); |
686 | |
687 | #ifdef CONFIG_STACKTRACE |
688 | entry->stack_len = stack_trace_save(store: entry->stack_entries, |
689 | ARRAY_SIZE(entry->stack_entries), |
690 | skipnr: 1); |
691 | #endif |
692 | return entry; |
693 | } |
694 | |
695 | static void dma_entry_free(struct dma_debug_entry *entry) |
696 | { |
697 | unsigned long flags; |
698 | |
699 | active_cacheline_remove(entry); |
700 | |
701 | /* |
702 | * add to beginning of the list - this way the entries are |
703 | * more likely cache hot when they are reallocated. |
704 | */ |
705 | spin_lock_irqsave(&free_entries_lock, flags); |
706 | list_add(new: &entry->list, head: &free_entries); |
707 | num_free_entries += 1; |
708 | spin_unlock_irqrestore(lock: &free_entries_lock, flags); |
709 | } |
710 | |
711 | /* |
712 | * DMA-API debugging init code |
713 | * |
714 | * The init code does two things: |
715 | * 1. Initialize core data structures |
716 | * 2. Preallocate a given number of dma_debug_entry structs |
717 | */ |
718 | |
719 | static ssize_t filter_read(struct file *file, char __user *user_buf, |
720 | size_t count, loff_t *ppos) |
721 | { |
722 | char buf[NAME_MAX_LEN + 1]; |
723 | unsigned long flags; |
724 | int len; |
725 | |
726 | if (!current_driver_name[0]) |
727 | return 0; |
728 | |
729 | /* |
730 | * We can't copy to userspace directly because current_driver_name can |
731 | * only be read under the driver_name_lock with irqs disabled. So |
732 | * create a temporary copy first. |
733 | */ |
734 | read_lock_irqsave(&driver_name_lock, flags); |
735 | len = scnprintf(buf, NAME_MAX_LEN + 1, fmt: "%s\n" , current_driver_name); |
736 | read_unlock_irqrestore(&driver_name_lock, flags); |
737 | |
738 | return simple_read_from_buffer(to: user_buf, count, ppos, from: buf, available: len); |
739 | } |
740 | |
741 | static ssize_t filter_write(struct file *file, const char __user *userbuf, |
742 | size_t count, loff_t *ppos) |
743 | { |
744 | char buf[NAME_MAX_LEN]; |
745 | unsigned long flags; |
746 | size_t len; |
747 | int i; |
748 | |
749 | /* |
750 | * We can't copy from userspace directly. Access to |
751 | * current_driver_name is protected with a write_lock with irqs |
752 | * disabled. Since copy_from_user can fault and may sleep we |
753 | * need to copy to temporary buffer first |
754 | */ |
755 | len = min(count, (size_t)(NAME_MAX_LEN - 1)); |
756 | if (copy_from_user(to: buf, from: userbuf, n: len)) |
757 | return -EFAULT; |
758 | |
759 | buf[len] = 0; |
760 | |
761 | write_lock_irqsave(&driver_name_lock, flags); |
762 | |
763 | /* |
764 | * Now handle the string we got from userspace very carefully. |
765 | * The rules are: |
766 | * - only use the first token we got |
767 | * - token delimiter is everything looking like a space |
768 | * character (' ', '\n', '\t' ...) |
769 | * |
770 | */ |
771 | if (!isalnum(buf[0])) { |
772 | /* |
773 | * If the first character userspace gave us is not |
774 | * alphanumerical then assume the filter should be |
775 | * switched off. |
776 | */ |
777 | if (current_driver_name[0]) |
778 | pr_info("switching off dma-debug driver filter\n" ); |
779 | current_driver_name[0] = 0; |
780 | current_driver = NULL; |
781 | goto out_unlock; |
782 | } |
783 | |
784 | /* |
785 | * Now parse out the first token and use it as the name for the |
786 | * driver to filter for. |
787 | */ |
788 | for (i = 0; i < NAME_MAX_LEN - 1; ++i) { |
789 | current_driver_name[i] = buf[i]; |
790 | if (isspace(buf[i]) || buf[i] == ' ' || buf[i] == 0) |
791 | break; |
792 | } |
793 | current_driver_name[i] = 0; |
794 | current_driver = NULL; |
795 | |
796 | pr_info("enable driver filter for driver [%s]\n" , |
797 | current_driver_name); |
798 | |
799 | out_unlock: |
800 | write_unlock_irqrestore(&driver_name_lock, flags); |
801 | |
802 | return count; |
803 | } |
804 | |
805 | static const struct file_operations filter_fops = { |
806 | .read = filter_read, |
807 | .write = filter_write, |
808 | .llseek = default_llseek, |
809 | }; |
810 | |
811 | static int __init dma_debug_fs_init(void) |
812 | { |
813 | struct dentry *dentry = debugfs_create_dir(name: "dma-api" , NULL); |
814 | |
815 | debugfs_create_bool(name: "disabled" , mode: 0444, parent: dentry, value: &global_disable); |
816 | debugfs_create_u32(name: "error_count" , mode: 0444, parent: dentry, value: &error_count); |
817 | debugfs_create_u32(name: "all_errors" , mode: 0644, parent: dentry, value: &show_all_errors); |
818 | debugfs_create_u32(name: "num_errors" , mode: 0644, parent: dentry, value: &show_num_errors); |
819 | debugfs_create_u32(name: "num_free_entries" , mode: 0444, parent: dentry, value: &num_free_entries); |
820 | debugfs_create_u32(name: "min_free_entries" , mode: 0444, parent: dentry, value: &min_free_entries); |
821 | debugfs_create_u32(name: "nr_total_entries" , mode: 0444, parent: dentry, value: &nr_total_entries); |
822 | debugfs_create_file(name: "driver_filter" , mode: 0644, parent: dentry, NULL, fops: &filter_fops); |
823 | debugfs_create_file(name: "dump" , mode: 0444, parent: dentry, NULL, fops: &dump_fops); |
824 | |
825 | return 0; |
826 | } |
827 | core_initcall_sync(dma_debug_fs_init); |
828 | |
829 | static int device_dma_allocations(struct device *dev, struct dma_debug_entry **out_entry) |
830 | { |
831 | struct dma_debug_entry *entry; |
832 | unsigned long flags; |
833 | int count = 0, i; |
834 | |
835 | for (i = 0; i < HASH_SIZE; ++i) { |
836 | spin_lock_irqsave(&dma_entry_hash[i].lock, flags); |
837 | list_for_each_entry(entry, &dma_entry_hash[i].list, list) { |
838 | if (entry->dev == dev) { |
839 | count += 1; |
840 | *out_entry = entry; |
841 | } |
842 | } |
843 | spin_unlock_irqrestore(lock: &dma_entry_hash[i].lock, flags); |
844 | } |
845 | |
846 | return count; |
847 | } |
848 | |
849 | static int dma_debug_device_change(struct notifier_block *nb, unsigned long action, void *data) |
850 | { |
851 | struct device *dev = data; |
852 | struct dma_debug_entry *entry; |
853 | int count; |
854 | |
855 | if (dma_debug_disabled()) |
856 | return 0; |
857 | |
858 | switch (action) { |
859 | case BUS_NOTIFY_UNBOUND_DRIVER: |
860 | count = device_dma_allocations(dev, out_entry: &entry); |
861 | if (count == 0) |
862 | break; |
863 | err_printk(dev, entry, "device driver has pending " |
864 | "DMA allocations while released from device " |
865 | "[count=%d]\n" |
866 | "One of leaked entries details: " |
867 | "[device address=0x%016llx] [size=%llu bytes] " |
868 | "[mapped with %s] [mapped as %s]\n" , |
869 | count, entry->dev_addr, entry->size, |
870 | dir2name[entry->direction], type2name[entry->type]); |
871 | break; |
872 | default: |
873 | break; |
874 | } |
875 | |
876 | return 0; |
877 | } |
878 | |
879 | void dma_debug_add_bus(struct bus_type *bus) |
880 | { |
881 | struct notifier_block *nb; |
882 | |
883 | if (dma_debug_disabled()) |
884 | return; |
885 | |
886 | nb = kzalloc(size: sizeof(struct notifier_block), GFP_KERNEL); |
887 | if (nb == NULL) { |
888 | pr_err("dma_debug_add_bus: out of memory\n" ); |
889 | return; |
890 | } |
891 | |
892 | nb->notifier_call = dma_debug_device_change; |
893 | |
894 | bus_register_notifier(bus, nb); |
895 | } |
896 | |
897 | static int dma_debug_init(void) |
898 | { |
899 | int i, nr_pages; |
900 | |
901 | /* Do not use dma_debug_initialized here, since we really want to be |
902 | * called to set dma_debug_initialized |
903 | */ |
904 | if (global_disable) |
905 | return 0; |
906 | |
907 | for (i = 0; i < HASH_SIZE; ++i) { |
908 | INIT_LIST_HEAD(list: &dma_entry_hash[i].list); |
909 | spin_lock_init(&dma_entry_hash[i].lock); |
910 | } |
911 | |
912 | nr_pages = DIV_ROUND_UP(nr_prealloc_entries, DMA_DEBUG_DYNAMIC_ENTRIES); |
913 | for (i = 0; i < nr_pages; ++i) |
914 | dma_debug_create_entries(GFP_KERNEL); |
915 | if (num_free_entries >= nr_prealloc_entries) { |
916 | pr_info("preallocated %d debug entries\n" , nr_total_entries); |
917 | } else if (num_free_entries > 0) { |
918 | pr_warn("%d debug entries requested but only %d allocated\n" , |
919 | nr_prealloc_entries, nr_total_entries); |
920 | } else { |
921 | pr_err("debugging out of memory error - disabled\n" ); |
922 | global_disable = true; |
923 | |
924 | return 0; |
925 | } |
926 | min_free_entries = num_free_entries; |
927 | |
928 | dma_debug_initialized = true; |
929 | |
930 | pr_info("debugging enabled by kernel config\n" ); |
931 | return 0; |
932 | } |
933 | core_initcall(dma_debug_init); |
934 | |
935 | static __init int dma_debug_cmdline(char *str) |
936 | { |
937 | if (!str) |
938 | return -EINVAL; |
939 | |
940 | if (strncmp(str, "off" , 3) == 0) { |
941 | pr_info("debugging disabled on kernel command line\n" ); |
942 | global_disable = true; |
943 | } |
944 | |
945 | return 1; |
946 | } |
947 | |
948 | static __init int dma_debug_entries_cmdline(char *str) |
949 | { |
950 | if (!str) |
951 | return -EINVAL; |
952 | if (!get_option(str: &str, pint: &nr_prealloc_entries)) |
953 | nr_prealloc_entries = PREALLOC_DMA_DEBUG_ENTRIES; |
954 | return 1; |
955 | } |
956 | |
957 | __setup("dma_debug=" , dma_debug_cmdline); |
958 | __setup("dma_debug_entries=" , dma_debug_entries_cmdline); |
959 | |
960 | static void check_unmap(struct dma_debug_entry *ref) |
961 | { |
962 | struct dma_debug_entry *entry; |
963 | struct hash_bucket *bucket; |
964 | unsigned long flags; |
965 | |
966 | bucket = get_hash_bucket(entry: ref, flags: &flags); |
967 | entry = bucket_find_exact(bucket, ref); |
968 | |
969 | if (!entry) { |
970 | /* must drop lock before calling dma_mapping_error */ |
971 | put_hash_bucket(bucket, flags); |
972 | |
973 | if (dma_mapping_error(dev: ref->dev, dma_addr: ref->dev_addr)) { |
974 | err_printk(ref->dev, NULL, |
975 | "device driver tries to free an " |
976 | "invalid DMA memory address\n" ); |
977 | } else { |
978 | err_printk(ref->dev, NULL, |
979 | "device driver tries to free DMA " |
980 | "memory it has not allocated [device " |
981 | "address=0x%016llx] [size=%llu bytes]\n" , |
982 | ref->dev_addr, ref->size); |
983 | } |
984 | return; |
985 | } |
986 | |
987 | if (ref->size != entry->size) { |
988 | err_printk(ref->dev, entry, "device driver frees " |
989 | "DMA memory with different size " |
990 | "[device address=0x%016llx] [map size=%llu bytes] " |
991 | "[unmap size=%llu bytes]\n" , |
992 | ref->dev_addr, entry->size, ref->size); |
993 | } |
994 | |
995 | if (ref->type != entry->type) { |
996 | err_printk(ref->dev, entry, "device driver frees " |
997 | "DMA memory with wrong function " |
998 | "[device address=0x%016llx] [size=%llu bytes] " |
999 | "[mapped as %s] [unmapped as %s]\n" , |
1000 | ref->dev_addr, ref->size, |
1001 | type2name[entry->type], type2name[ref->type]); |
1002 | } else if ((entry->type == dma_debug_coherent) && |
1003 | (phys_addr(entry: ref) != phys_addr(entry))) { |
1004 | err_printk(ref->dev, entry, "device driver frees " |
1005 | "DMA memory with different CPU address " |
1006 | "[device address=0x%016llx] [size=%llu bytes] " |
1007 | "[cpu alloc address=0x%016llx] " |
1008 | "[cpu free address=0x%016llx]" , |
1009 | ref->dev_addr, ref->size, |
1010 | phys_addr(entry), |
1011 | phys_addr(ref)); |
1012 | } |
1013 | |
1014 | if (ref->sg_call_ents && ref->type == dma_debug_sg && |
1015 | ref->sg_call_ents != entry->sg_call_ents) { |
1016 | err_printk(ref->dev, entry, "device driver frees " |
1017 | "DMA sg list with different entry count " |
1018 | "[map count=%d] [unmap count=%d]\n" , |
1019 | entry->sg_call_ents, ref->sg_call_ents); |
1020 | } |
1021 | |
1022 | /* |
1023 | * This may be no bug in reality - but most implementations of the |
1024 | * DMA API don't handle this properly, so check for it here |
1025 | */ |
1026 | if (ref->direction != entry->direction) { |
1027 | err_printk(ref->dev, entry, "device driver frees " |
1028 | "DMA memory with different direction " |
1029 | "[device address=0x%016llx] [size=%llu bytes] " |
1030 | "[mapped with %s] [unmapped with %s]\n" , |
1031 | ref->dev_addr, ref->size, |
1032 | dir2name[entry->direction], |
1033 | dir2name[ref->direction]); |
1034 | } |
1035 | |
1036 | /* |
1037 | * Drivers should use dma_mapping_error() to check the returned |
1038 | * addresses of dma_map_single() and dma_map_page(). |
1039 | * If not, print this warning message. See Documentation/core-api/dma-api.rst. |
1040 | */ |
1041 | if (entry->map_err_type == MAP_ERR_NOT_CHECKED) { |
1042 | err_printk(ref->dev, entry, |
1043 | "device driver failed to check map error" |
1044 | "[device address=0x%016llx] [size=%llu bytes] " |
1045 | "[mapped as %s]" , |
1046 | ref->dev_addr, ref->size, |
1047 | type2name[entry->type]); |
1048 | } |
1049 | |
1050 | hash_bucket_del(entry); |
1051 | dma_entry_free(entry); |
1052 | |
1053 | put_hash_bucket(bucket, flags); |
1054 | } |
1055 | |
1056 | static void check_for_stack(struct device *dev, |
1057 | struct page *page, size_t offset) |
1058 | { |
1059 | void *addr; |
1060 | struct vm_struct *stack_vm_area = task_stack_vm_area(current); |
1061 | |
1062 | if (!stack_vm_area) { |
1063 | /* Stack is direct-mapped. */ |
1064 | if (PageHighMem(page)) |
1065 | return; |
1066 | addr = page_address(page) + offset; |
1067 | if (object_is_on_stack(obj: addr)) |
1068 | err_printk(dev, NULL, "device driver maps memory from stack [addr=%p]\n" , addr); |
1069 | } else { |
1070 | /* Stack is vmalloced. */ |
1071 | int i; |
1072 | |
1073 | for (i = 0; i < stack_vm_area->nr_pages; i++) { |
1074 | if (page != stack_vm_area->pages[i]) |
1075 | continue; |
1076 | |
1077 | addr = (u8 *)current->stack + i * PAGE_SIZE + offset; |
1078 | err_printk(dev, NULL, "device driver maps memory from stack [probable addr=%p]\n" , addr); |
1079 | break; |
1080 | } |
1081 | } |
1082 | } |
1083 | |
1084 | static void check_for_illegal_area(struct device *dev, void *addr, unsigned long len) |
1085 | { |
1086 | if (memory_intersects(begin: _stext, end: _etext, virt: addr, size: len) || |
1087 | memory_intersects(begin: __start_rodata, end: __end_rodata, virt: addr, size: len)) |
1088 | err_printk(dev, NULL, "device driver maps memory from kernel text or rodata [addr=%p] [len=%lu]\n" , addr, len); |
1089 | } |
1090 | |
1091 | static void check_sync(struct device *dev, |
1092 | struct dma_debug_entry *ref, |
1093 | bool to_cpu) |
1094 | { |
1095 | struct dma_debug_entry *entry; |
1096 | struct hash_bucket *bucket; |
1097 | unsigned long flags; |
1098 | |
1099 | bucket = get_hash_bucket(entry: ref, flags: &flags); |
1100 | |
1101 | entry = bucket_find_contain(bucket: &bucket, ref, flags: &flags); |
1102 | |
1103 | if (!entry) { |
1104 | err_printk(dev, NULL, "device driver tries " |
1105 | "to sync DMA memory it has not allocated " |
1106 | "[device address=0x%016llx] [size=%llu bytes]\n" , |
1107 | (unsigned long long)ref->dev_addr, ref->size); |
1108 | goto out; |
1109 | } |
1110 | |
1111 | if (ref->size > entry->size) { |
1112 | err_printk(dev, entry, "device driver syncs" |
1113 | " DMA memory outside allocated range " |
1114 | "[device address=0x%016llx] " |
1115 | "[allocation size=%llu bytes] " |
1116 | "[sync offset+size=%llu]\n" , |
1117 | entry->dev_addr, entry->size, |
1118 | ref->size); |
1119 | } |
1120 | |
1121 | if (entry->direction == DMA_BIDIRECTIONAL) |
1122 | goto out; |
1123 | |
1124 | if (ref->direction != entry->direction) { |
1125 | err_printk(dev, entry, "device driver syncs " |
1126 | "DMA memory with different direction " |
1127 | "[device address=0x%016llx] [size=%llu bytes] " |
1128 | "[mapped with %s] [synced with %s]\n" , |
1129 | (unsigned long long)ref->dev_addr, entry->size, |
1130 | dir2name[entry->direction], |
1131 | dir2name[ref->direction]); |
1132 | } |
1133 | |
1134 | if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) && |
1135 | !(ref->direction == DMA_TO_DEVICE)) |
1136 | err_printk(dev, entry, "device driver syncs " |
1137 | "device read-only DMA memory for cpu " |
1138 | "[device address=0x%016llx] [size=%llu bytes] " |
1139 | "[mapped with %s] [synced with %s]\n" , |
1140 | (unsigned long long)ref->dev_addr, entry->size, |
1141 | dir2name[entry->direction], |
1142 | dir2name[ref->direction]); |
1143 | |
1144 | if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) && |
1145 | !(ref->direction == DMA_FROM_DEVICE)) |
1146 | err_printk(dev, entry, "device driver syncs " |
1147 | "device write-only DMA memory to device " |
1148 | "[device address=0x%016llx] [size=%llu bytes] " |
1149 | "[mapped with %s] [synced with %s]\n" , |
1150 | (unsigned long long)ref->dev_addr, entry->size, |
1151 | dir2name[entry->direction], |
1152 | dir2name[ref->direction]); |
1153 | |
1154 | if (ref->sg_call_ents && ref->type == dma_debug_sg && |
1155 | ref->sg_call_ents != entry->sg_call_ents) { |
1156 | err_printk(ref->dev, entry, "device driver syncs " |
1157 | "DMA sg list with different entry count " |
1158 | "[map count=%d] [sync count=%d]\n" , |
1159 | entry->sg_call_ents, ref->sg_call_ents); |
1160 | } |
1161 | |
1162 | out: |
1163 | put_hash_bucket(bucket, flags); |
1164 | } |
1165 | |
1166 | static void check_sg_segment(struct device *dev, struct scatterlist *sg) |
1167 | { |
1168 | #ifdef CONFIG_DMA_API_DEBUG_SG |
1169 | unsigned int max_seg = dma_get_max_seg_size(dev); |
1170 | u64 start, end, boundary = dma_get_seg_boundary(dev); |
1171 | |
1172 | /* |
1173 | * Either the driver forgot to set dma_parms appropriately, or |
1174 | * whoever generated the list forgot to check them. |
1175 | */ |
1176 | if (sg->length > max_seg) |
1177 | err_printk(dev, NULL, "mapping sg segment longer than device claims to support [len=%u] [max=%u]\n" , |
1178 | sg->length, max_seg); |
1179 | /* |
1180 | * In some cases this could potentially be the DMA API |
1181 | * implementation's fault, but it would usually imply that |
1182 | * the scatterlist was built inappropriately to begin with. |
1183 | */ |
1184 | start = sg_dma_address(sg); |
1185 | end = start + sg_dma_len(sg) - 1; |
1186 | if ((start ^ end) & ~boundary) |
1187 | err_printk(dev, NULL, "mapping sg segment across boundary [start=0x%016llx] [end=0x%016llx] [boundary=0x%016llx]\n" , |
1188 | start, end, boundary); |
1189 | #endif |
1190 | } |
1191 | |
1192 | void debug_dma_map_single(struct device *dev, const void *addr, |
1193 | unsigned long len) |
1194 | { |
1195 | if (unlikely(dma_debug_disabled())) |
1196 | return; |
1197 | |
1198 | if (!virt_addr_valid(addr)) |
1199 | err_printk(dev, NULL, "device driver maps memory from invalid area [addr=%p] [len=%lu]\n" , |
1200 | addr, len); |
1201 | |
1202 | if (is_vmalloc_addr(x: addr)) |
1203 | err_printk(dev, NULL, "device driver maps memory from vmalloc area [addr=%p] [len=%lu]\n" , |
1204 | addr, len); |
1205 | } |
1206 | EXPORT_SYMBOL(debug_dma_map_single); |
1207 | |
1208 | void debug_dma_map_page(struct device *dev, struct page *page, size_t offset, |
1209 | size_t size, int direction, dma_addr_t dma_addr, |
1210 | unsigned long attrs) |
1211 | { |
1212 | struct dma_debug_entry *entry; |
1213 | |
1214 | if (unlikely(dma_debug_disabled())) |
1215 | return; |
1216 | |
1217 | if (dma_mapping_error(dev, dma_addr)) |
1218 | return; |
1219 | |
1220 | entry = dma_entry_alloc(); |
1221 | if (!entry) |
1222 | return; |
1223 | |
1224 | entry->dev = dev; |
1225 | entry->type = dma_debug_single; |
1226 | entry->pfn = page_to_pfn(page); |
1227 | entry->offset = offset; |
1228 | entry->dev_addr = dma_addr; |
1229 | entry->size = size; |
1230 | entry->direction = direction; |
1231 | entry->map_err_type = MAP_ERR_NOT_CHECKED; |
1232 | |
1233 | check_for_stack(dev, page, offset); |
1234 | |
1235 | if (!PageHighMem(page)) { |
1236 | void *addr = page_address(page) + offset; |
1237 | |
1238 | check_for_illegal_area(dev, addr, len: size); |
1239 | } |
1240 | |
1241 | add_dma_entry(entry, attrs); |
1242 | } |
1243 | |
1244 | void debug_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) |
1245 | { |
1246 | struct dma_debug_entry ref; |
1247 | struct dma_debug_entry *entry; |
1248 | struct hash_bucket *bucket; |
1249 | unsigned long flags; |
1250 | |
1251 | if (unlikely(dma_debug_disabled())) |
1252 | return; |
1253 | |
1254 | ref.dev = dev; |
1255 | ref.dev_addr = dma_addr; |
1256 | bucket = get_hash_bucket(entry: &ref, flags: &flags); |
1257 | |
1258 | list_for_each_entry(entry, &bucket->list, list) { |
1259 | if (!exact_match(a: &ref, b: entry)) |
1260 | continue; |
1261 | |
1262 | /* |
1263 | * The same physical address can be mapped multiple |
1264 | * times. Without a hardware IOMMU this results in the |
1265 | * same device addresses being put into the dma-debug |
1266 | * hash multiple times too. This can result in false |
1267 | * positives being reported. Therefore we implement a |
1268 | * best-fit algorithm here which updates the first entry |
1269 | * from the hash which fits the reference value and is |
1270 | * not currently listed as being checked. |
1271 | */ |
1272 | if (entry->map_err_type == MAP_ERR_NOT_CHECKED) { |
1273 | entry->map_err_type = MAP_ERR_CHECKED; |
1274 | break; |
1275 | } |
1276 | } |
1277 | |
1278 | put_hash_bucket(bucket, flags); |
1279 | } |
1280 | EXPORT_SYMBOL(debug_dma_mapping_error); |
1281 | |
1282 | void debug_dma_unmap_page(struct device *dev, dma_addr_t dma_addr, |
1283 | size_t size, int direction) |
1284 | { |
1285 | struct dma_debug_entry ref = { |
1286 | .type = dma_debug_single, |
1287 | .dev = dev, |
1288 | .dev_addr = dma_addr, |
1289 | .size = size, |
1290 | .direction = direction, |
1291 | }; |
1292 | |
1293 | if (unlikely(dma_debug_disabled())) |
1294 | return; |
1295 | check_unmap(ref: &ref); |
1296 | } |
1297 | |
1298 | void debug_dma_map_sg(struct device *dev, struct scatterlist *sg, |
1299 | int nents, int mapped_ents, int direction, |
1300 | unsigned long attrs) |
1301 | { |
1302 | struct dma_debug_entry *entry; |
1303 | struct scatterlist *s; |
1304 | int i; |
1305 | |
1306 | if (unlikely(dma_debug_disabled())) |
1307 | return; |
1308 | |
1309 | for_each_sg(sg, s, nents, i) { |
1310 | check_for_stack(dev, page: sg_page(sg: s), offset: s->offset); |
1311 | if (!PageHighMem(page: sg_page(sg: s))) |
1312 | check_for_illegal_area(dev, addr: sg_virt(sg: s), len: s->length); |
1313 | } |
1314 | |
1315 | for_each_sg(sg, s, mapped_ents, i) { |
1316 | entry = dma_entry_alloc(); |
1317 | if (!entry) |
1318 | return; |
1319 | |
1320 | entry->type = dma_debug_sg; |
1321 | entry->dev = dev; |
1322 | entry->pfn = page_to_pfn(sg_page(s)); |
1323 | entry->offset = s->offset; |
1324 | entry->size = sg_dma_len(s); |
1325 | entry->dev_addr = sg_dma_address(s); |
1326 | entry->direction = direction; |
1327 | entry->sg_call_ents = nents; |
1328 | entry->sg_mapped_ents = mapped_ents; |
1329 | |
1330 | check_sg_segment(dev, sg: s); |
1331 | |
1332 | add_dma_entry(entry, attrs); |
1333 | } |
1334 | } |
1335 | |
1336 | static int get_nr_mapped_entries(struct device *dev, |
1337 | struct dma_debug_entry *ref) |
1338 | { |
1339 | struct dma_debug_entry *entry; |
1340 | struct hash_bucket *bucket; |
1341 | unsigned long flags; |
1342 | int mapped_ents; |
1343 | |
1344 | bucket = get_hash_bucket(entry: ref, flags: &flags); |
1345 | entry = bucket_find_exact(bucket, ref); |
1346 | mapped_ents = 0; |
1347 | |
1348 | if (entry) |
1349 | mapped_ents = entry->sg_mapped_ents; |
1350 | put_hash_bucket(bucket, flags); |
1351 | |
1352 | return mapped_ents; |
1353 | } |
1354 | |
1355 | void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, |
1356 | int nelems, int dir) |
1357 | { |
1358 | struct scatterlist *s; |
1359 | int mapped_ents = 0, i; |
1360 | |
1361 | if (unlikely(dma_debug_disabled())) |
1362 | return; |
1363 | |
1364 | for_each_sg(sglist, s, nelems, i) { |
1365 | |
1366 | struct dma_debug_entry ref = { |
1367 | .type = dma_debug_sg, |
1368 | .dev = dev, |
1369 | .pfn = page_to_pfn(sg_page(s)), |
1370 | .offset = s->offset, |
1371 | .dev_addr = sg_dma_address(s), |
1372 | .size = sg_dma_len(s), |
1373 | .direction = dir, |
1374 | .sg_call_ents = nelems, |
1375 | }; |
1376 | |
1377 | if (mapped_ents && i >= mapped_ents) |
1378 | break; |
1379 | |
1380 | if (!i) |
1381 | mapped_ents = get_nr_mapped_entries(dev, ref: &ref); |
1382 | |
1383 | check_unmap(ref: &ref); |
1384 | } |
1385 | } |
1386 | |
1387 | void debug_dma_alloc_coherent(struct device *dev, size_t size, |
1388 | dma_addr_t dma_addr, void *virt, |
1389 | unsigned long attrs) |
1390 | { |
1391 | struct dma_debug_entry *entry; |
1392 | |
1393 | if (unlikely(dma_debug_disabled())) |
1394 | return; |
1395 | |
1396 | if (unlikely(virt == NULL)) |
1397 | return; |
1398 | |
1399 | /* handle vmalloc and linear addresses */ |
1400 | if (!is_vmalloc_addr(x: virt) && !virt_addr_valid(virt)) |
1401 | return; |
1402 | |
1403 | entry = dma_entry_alloc(); |
1404 | if (!entry) |
1405 | return; |
1406 | |
1407 | entry->type = dma_debug_coherent; |
1408 | entry->dev = dev; |
1409 | entry->offset = offset_in_page(virt); |
1410 | entry->size = size; |
1411 | entry->dev_addr = dma_addr; |
1412 | entry->direction = DMA_BIDIRECTIONAL; |
1413 | |
1414 | if (is_vmalloc_addr(x: virt)) |
1415 | entry->pfn = vmalloc_to_pfn(addr: virt); |
1416 | else |
1417 | entry->pfn = page_to_pfn(virt_to_page(virt)); |
1418 | |
1419 | add_dma_entry(entry, attrs); |
1420 | } |
1421 | |
1422 | void debug_dma_free_coherent(struct device *dev, size_t size, |
1423 | void *virt, dma_addr_t dma_addr) |
1424 | { |
1425 | struct dma_debug_entry ref = { |
1426 | .type = dma_debug_coherent, |
1427 | .dev = dev, |
1428 | .offset = offset_in_page(virt), |
1429 | .dev_addr = dma_addr, |
1430 | .size = size, |
1431 | .direction = DMA_BIDIRECTIONAL, |
1432 | }; |
1433 | |
1434 | /* handle vmalloc and linear addresses */ |
1435 | if (!is_vmalloc_addr(x: virt) && !virt_addr_valid(virt)) |
1436 | return; |
1437 | |
1438 | if (is_vmalloc_addr(x: virt)) |
1439 | ref.pfn = vmalloc_to_pfn(addr: virt); |
1440 | else |
1441 | ref.pfn = page_to_pfn(virt_to_page(virt)); |
1442 | |
1443 | if (unlikely(dma_debug_disabled())) |
1444 | return; |
1445 | |
1446 | check_unmap(ref: &ref); |
1447 | } |
1448 | |
1449 | void debug_dma_map_resource(struct device *dev, phys_addr_t addr, size_t size, |
1450 | int direction, dma_addr_t dma_addr, |
1451 | unsigned long attrs) |
1452 | { |
1453 | struct dma_debug_entry *entry; |
1454 | |
1455 | if (unlikely(dma_debug_disabled())) |
1456 | return; |
1457 | |
1458 | entry = dma_entry_alloc(); |
1459 | if (!entry) |
1460 | return; |
1461 | |
1462 | entry->type = dma_debug_resource; |
1463 | entry->dev = dev; |
1464 | entry->pfn = PHYS_PFN(addr); |
1465 | entry->offset = offset_in_page(addr); |
1466 | entry->size = size; |
1467 | entry->dev_addr = dma_addr; |
1468 | entry->direction = direction; |
1469 | entry->map_err_type = MAP_ERR_NOT_CHECKED; |
1470 | |
1471 | add_dma_entry(entry, attrs); |
1472 | } |
1473 | |
1474 | void debug_dma_unmap_resource(struct device *dev, dma_addr_t dma_addr, |
1475 | size_t size, int direction) |
1476 | { |
1477 | struct dma_debug_entry ref = { |
1478 | .type = dma_debug_resource, |
1479 | .dev = dev, |
1480 | .dev_addr = dma_addr, |
1481 | .size = size, |
1482 | .direction = direction, |
1483 | }; |
1484 | |
1485 | if (unlikely(dma_debug_disabled())) |
1486 | return; |
1487 | |
1488 | check_unmap(ref: &ref); |
1489 | } |
1490 | |
1491 | void debug_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, |
1492 | size_t size, int direction) |
1493 | { |
1494 | struct dma_debug_entry ref; |
1495 | |
1496 | if (unlikely(dma_debug_disabled())) |
1497 | return; |
1498 | |
1499 | ref.type = dma_debug_single; |
1500 | ref.dev = dev; |
1501 | ref.dev_addr = dma_handle; |
1502 | ref.size = size; |
1503 | ref.direction = direction; |
1504 | ref.sg_call_ents = 0; |
1505 | |
1506 | check_sync(dev, ref: &ref, to_cpu: true); |
1507 | } |
1508 | |
1509 | void debug_dma_sync_single_for_device(struct device *dev, |
1510 | dma_addr_t dma_handle, size_t size, |
1511 | int direction) |
1512 | { |
1513 | struct dma_debug_entry ref; |
1514 | |
1515 | if (unlikely(dma_debug_disabled())) |
1516 | return; |
1517 | |
1518 | ref.type = dma_debug_single; |
1519 | ref.dev = dev; |
1520 | ref.dev_addr = dma_handle; |
1521 | ref.size = size; |
1522 | ref.direction = direction; |
1523 | ref.sg_call_ents = 0; |
1524 | |
1525 | check_sync(dev, ref: &ref, to_cpu: false); |
1526 | } |
1527 | |
1528 | void debug_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, |
1529 | int nelems, int direction) |
1530 | { |
1531 | struct scatterlist *s; |
1532 | int mapped_ents = 0, i; |
1533 | |
1534 | if (unlikely(dma_debug_disabled())) |
1535 | return; |
1536 | |
1537 | for_each_sg(sg, s, nelems, i) { |
1538 | |
1539 | struct dma_debug_entry ref = { |
1540 | .type = dma_debug_sg, |
1541 | .dev = dev, |
1542 | .pfn = page_to_pfn(sg_page(s)), |
1543 | .offset = s->offset, |
1544 | .dev_addr = sg_dma_address(s), |
1545 | .size = sg_dma_len(s), |
1546 | .direction = direction, |
1547 | .sg_call_ents = nelems, |
1548 | }; |
1549 | |
1550 | if (!i) |
1551 | mapped_ents = get_nr_mapped_entries(dev, ref: &ref); |
1552 | |
1553 | if (i >= mapped_ents) |
1554 | break; |
1555 | |
1556 | check_sync(dev, ref: &ref, to_cpu: true); |
1557 | } |
1558 | } |
1559 | |
1560 | void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, |
1561 | int nelems, int direction) |
1562 | { |
1563 | struct scatterlist *s; |
1564 | int mapped_ents = 0, i; |
1565 | |
1566 | if (unlikely(dma_debug_disabled())) |
1567 | return; |
1568 | |
1569 | for_each_sg(sg, s, nelems, i) { |
1570 | |
1571 | struct dma_debug_entry ref = { |
1572 | .type = dma_debug_sg, |
1573 | .dev = dev, |
1574 | .pfn = page_to_pfn(sg_page(s)), |
1575 | .offset = s->offset, |
1576 | .dev_addr = sg_dma_address(s), |
1577 | .size = sg_dma_len(s), |
1578 | .direction = direction, |
1579 | .sg_call_ents = nelems, |
1580 | }; |
1581 | if (!i) |
1582 | mapped_ents = get_nr_mapped_entries(dev, ref: &ref); |
1583 | |
1584 | if (i >= mapped_ents) |
1585 | break; |
1586 | |
1587 | check_sync(dev, ref: &ref, to_cpu: false); |
1588 | } |
1589 | } |
1590 | |
1591 | static int __init dma_debug_driver_setup(char *str) |
1592 | { |
1593 | int i; |
1594 | |
1595 | for (i = 0; i < NAME_MAX_LEN - 1; ++i, ++str) { |
1596 | current_driver_name[i] = *str; |
1597 | if (*str == 0) |
1598 | break; |
1599 | } |
1600 | |
1601 | if (current_driver_name[0]) |
1602 | pr_info("enable driver filter for driver [%s]\n" , |
1603 | current_driver_name); |
1604 | |
1605 | |
1606 | return 1; |
1607 | } |
1608 | __setup("dma_debug_driver=" , dma_debug_driver_setup); |
1609 | |