1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Dynamic DMA mapping support for AMD Hammer. |
4 | * |
5 | * Use the integrated AGP GART in the Hammer northbridge as an IOMMU for PCI. |
6 | * This allows to use PCI devices that only support 32bit addresses on systems |
7 | * with more than 4GB. |
8 | * |
9 | * See Documentation/core-api/dma-api-howto.rst for the interface specification. |
10 | * |
11 | * Copyright 2002 Andi Kleen, SuSE Labs. |
12 | */ |
13 | |
14 | #include <linux/types.h> |
15 | #include <linux/ctype.h> |
16 | #include <linux/agp_backend.h> |
17 | #include <linux/init.h> |
18 | #include <linux/mm.h> |
19 | #include <linux/sched.h> |
20 | #include <linux/sched/debug.h> |
21 | #include <linux/string.h> |
22 | #include <linux/spinlock.h> |
23 | #include <linux/pci.h> |
24 | #include <linux/topology.h> |
25 | #include <linux/interrupt.h> |
26 | #include <linux/bitmap.h> |
27 | #include <linux/kdebug.h> |
28 | #include <linux/scatterlist.h> |
29 | #include <linux/iommu-helper.h> |
30 | #include <linux/syscore_ops.h> |
31 | #include <linux/io.h> |
32 | #include <linux/gfp.h> |
33 | #include <linux/atomic.h> |
34 | #include <linux/dma-direct.h> |
35 | #include <linux/dma-map-ops.h> |
36 | #include <asm/mtrr.h> |
37 | #include <asm/proto.h> |
38 | #include <asm/iommu.h> |
39 | #include <asm/gart.h> |
40 | #include <asm/set_memory.h> |
41 | #include <asm/dma.h> |
42 | #include <asm/amd_nb.h> |
43 | #include <asm/x86_init.h> |
44 | |
45 | static unsigned long iommu_bus_base; /* GART remapping area (physical) */ |
46 | static unsigned long iommu_size; /* size of remapping area bytes */ |
47 | static unsigned long iommu_pages; /* .. and in pages */ |
48 | |
49 | static u32 *iommu_gatt_base; /* Remapping table */ |
50 | |
51 | /* |
52 | * If this is disabled the IOMMU will use an optimized flushing strategy |
53 | * of only flushing when an mapping is reused. With it true the GART is |
54 | * flushed for every mapping. Problem is that doing the lazy flush seems |
55 | * to trigger bugs with some popular PCI cards, in particular 3ware (but |
56 | * has been also seen with Qlogic at least). |
57 | */ |
58 | static int iommu_fullflush = 1; |
59 | |
60 | /* Allocation bitmap for the remapping area: */ |
61 | static DEFINE_SPINLOCK(iommu_bitmap_lock); |
62 | /* Guarded by iommu_bitmap_lock: */ |
63 | static unsigned long *iommu_gart_bitmap; |
64 | |
65 | static u32 gart_unmapped_entry; |
66 | |
67 | #define GPTE_VALID 1 |
68 | #define GPTE_COHERENT 2 |
69 | #define GPTE_ENCODE(x) \ |
70 | (((x) & 0xfffff000) | (((x) >> 32) << 4) | GPTE_VALID | GPTE_COHERENT) |
71 | #define GPTE_DECODE(x) (((x) & 0xfffff000) | (((u64)(x) & 0xff0) << 28)) |
72 | |
73 | #ifdef CONFIG_AGP |
74 | #define AGPEXTERN extern |
75 | #else |
76 | #define AGPEXTERN |
77 | #endif |
78 | |
79 | /* GART can only remap to physical addresses < 1TB */ |
80 | #define GART_MAX_PHYS_ADDR (1ULL << 40) |
81 | |
82 | /* backdoor interface to AGP driver */ |
83 | AGPEXTERN int agp_memory_reserved; |
84 | AGPEXTERN __u32 *agp_gatt_table; |
85 | |
86 | static unsigned long next_bit; /* protected by iommu_bitmap_lock */ |
87 | static bool need_flush; /* global flush state. set for each gart wrap */ |
88 | |
89 | static unsigned long alloc_iommu(struct device *dev, int size, |
90 | unsigned long align_mask) |
91 | { |
92 | unsigned long offset, flags; |
93 | unsigned long boundary_size; |
94 | unsigned long base_index; |
95 | |
96 | base_index = ALIGN(iommu_bus_base & dma_get_seg_boundary(dev), |
97 | PAGE_SIZE) >> PAGE_SHIFT; |
98 | boundary_size = dma_get_seg_boundary_nr_pages(dev, PAGE_SHIFT); |
99 | |
100 | spin_lock_irqsave(&iommu_bitmap_lock, flags); |
101 | offset = iommu_area_alloc(map: iommu_gart_bitmap, size: iommu_pages, start: next_bit, |
102 | nr: size, shift: base_index, boundary_size, align_mask); |
103 | if (offset == -1) { |
104 | need_flush = true; |
105 | offset = iommu_area_alloc(map: iommu_gart_bitmap, size: iommu_pages, start: 0, |
106 | nr: size, shift: base_index, boundary_size, |
107 | align_mask); |
108 | } |
109 | if (offset != -1) { |
110 | next_bit = offset+size; |
111 | if (next_bit >= iommu_pages) { |
112 | next_bit = 0; |
113 | need_flush = true; |
114 | } |
115 | } |
116 | if (iommu_fullflush) |
117 | need_flush = true; |
118 | spin_unlock_irqrestore(lock: &iommu_bitmap_lock, flags); |
119 | |
120 | return offset; |
121 | } |
122 | |
123 | static void free_iommu(unsigned long offset, int size) |
124 | { |
125 | unsigned long flags; |
126 | |
127 | spin_lock_irqsave(&iommu_bitmap_lock, flags); |
128 | bitmap_clear(map: iommu_gart_bitmap, start: offset, nbits: size); |
129 | if (offset >= next_bit) |
130 | next_bit = offset + size; |
131 | spin_unlock_irqrestore(lock: &iommu_bitmap_lock, flags); |
132 | } |
133 | |
134 | /* |
135 | * Use global flush state to avoid races with multiple flushers. |
136 | */ |
137 | static void flush_gart(void) |
138 | { |
139 | unsigned long flags; |
140 | |
141 | spin_lock_irqsave(&iommu_bitmap_lock, flags); |
142 | if (need_flush) { |
143 | amd_flush_garts(); |
144 | need_flush = false; |
145 | } |
146 | spin_unlock_irqrestore(lock: &iommu_bitmap_lock, flags); |
147 | } |
148 | |
149 | #ifdef CONFIG_IOMMU_LEAK |
150 | /* Debugging aid for drivers that don't free their IOMMU tables */ |
151 | static void dump_leak(void) |
152 | { |
153 | static int dump; |
154 | |
155 | if (dump) |
156 | return; |
157 | dump = 1; |
158 | |
159 | show_stack(NULL, NULL, KERN_ERR); |
160 | debug_dma_dump_mappings(NULL); |
161 | } |
162 | #endif |
163 | |
164 | static void iommu_full(struct device *dev, size_t size, int dir) |
165 | { |
166 | /* |
167 | * Ran out of IOMMU space for this operation. This is very bad. |
168 | * Unfortunately the drivers cannot handle this operation properly. |
169 | * Return some non mapped prereserved space in the aperture and |
170 | * let the Northbridge deal with it. This will result in garbage |
171 | * in the IO operation. When the size exceeds the prereserved space |
172 | * memory corruption will occur or random memory will be DMAed |
173 | * out. Hopefully no network devices use single mappings that big. |
174 | */ |
175 | |
176 | dev_err(dev, "PCI-DMA: Out of IOMMU space for %lu bytes\n" , size); |
177 | #ifdef CONFIG_IOMMU_LEAK |
178 | dump_leak(); |
179 | #endif |
180 | } |
181 | |
182 | static inline int |
183 | need_iommu(struct device *dev, unsigned long addr, size_t size) |
184 | { |
185 | return force_iommu || !dma_capable(dev, addr, size, is_ram: true); |
186 | } |
187 | |
188 | static inline int |
189 | nonforced_iommu(struct device *dev, unsigned long addr, size_t size) |
190 | { |
191 | return !dma_capable(dev, addr, size, is_ram: true); |
192 | } |
193 | |
194 | /* Map a single continuous physical area into the IOMMU. |
195 | * Caller needs to check if the iommu is needed and flush. |
196 | */ |
197 | static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem, |
198 | size_t size, int dir, unsigned long align_mask) |
199 | { |
200 | unsigned long npages = iommu_num_pages(addr: phys_mem, len: size, PAGE_SIZE); |
201 | unsigned long iommu_page; |
202 | int i; |
203 | |
204 | if (unlikely(phys_mem + size > GART_MAX_PHYS_ADDR)) |
205 | return DMA_MAPPING_ERROR; |
206 | |
207 | iommu_page = alloc_iommu(dev, size: npages, align_mask); |
208 | if (iommu_page == -1) { |
209 | if (!nonforced_iommu(dev, addr: phys_mem, size)) |
210 | return phys_mem; |
211 | if (panic_on_overflow) |
212 | panic(fmt: "dma_map_area overflow %lu bytes\n" , size); |
213 | iommu_full(dev, size, dir); |
214 | return DMA_MAPPING_ERROR; |
215 | } |
216 | |
217 | for (i = 0; i < npages; i++) { |
218 | iommu_gatt_base[iommu_page + i] = GPTE_ENCODE(phys_mem); |
219 | phys_mem += PAGE_SIZE; |
220 | } |
221 | return iommu_bus_base + iommu_page*PAGE_SIZE + (phys_mem & ~PAGE_MASK); |
222 | } |
223 | |
224 | /* Map a single area into the IOMMU */ |
225 | static dma_addr_t gart_map_page(struct device *dev, struct page *page, |
226 | unsigned long offset, size_t size, |
227 | enum dma_data_direction dir, |
228 | unsigned long attrs) |
229 | { |
230 | unsigned long bus; |
231 | phys_addr_t paddr = page_to_phys(page) + offset; |
232 | |
233 | if (!need_iommu(dev, addr: paddr, size)) |
234 | return paddr; |
235 | |
236 | bus = dma_map_area(dev, phys_mem: paddr, size, dir, align_mask: 0); |
237 | flush_gart(); |
238 | |
239 | return bus; |
240 | } |
241 | |
242 | /* |
243 | * Free a DMA mapping. |
244 | */ |
245 | static void gart_unmap_page(struct device *dev, dma_addr_t dma_addr, |
246 | size_t size, enum dma_data_direction dir, |
247 | unsigned long attrs) |
248 | { |
249 | unsigned long iommu_page; |
250 | int npages; |
251 | int i; |
252 | |
253 | if (WARN_ON_ONCE(dma_addr == DMA_MAPPING_ERROR)) |
254 | return; |
255 | |
256 | /* |
257 | * This driver will not always use a GART mapping, but might have |
258 | * created a direct mapping instead. If that is the case there is |
259 | * nothing to unmap here. |
260 | */ |
261 | if (dma_addr < iommu_bus_base || |
262 | dma_addr >= iommu_bus_base + iommu_size) |
263 | return; |
264 | |
265 | iommu_page = (dma_addr - iommu_bus_base)>>PAGE_SHIFT; |
266 | npages = iommu_num_pages(addr: dma_addr, len: size, PAGE_SIZE); |
267 | for (i = 0; i < npages; i++) { |
268 | iommu_gatt_base[iommu_page + i] = gart_unmapped_entry; |
269 | } |
270 | free_iommu(offset: iommu_page, size: npages); |
271 | } |
272 | |
273 | /* |
274 | * Wrapper for pci_unmap_single working with scatterlists. |
275 | */ |
276 | static void gart_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, |
277 | enum dma_data_direction dir, unsigned long attrs) |
278 | { |
279 | struct scatterlist *s; |
280 | int i; |
281 | |
282 | for_each_sg(sg, s, nents, i) { |
283 | if (!s->dma_length || !s->length) |
284 | break; |
285 | gart_unmap_page(dev, dma_addr: s->dma_address, size: s->dma_length, dir, attrs: 0); |
286 | } |
287 | } |
288 | |
289 | /* Fallback for dma_map_sg in case of overflow */ |
290 | static int dma_map_sg_nonforce(struct device *dev, struct scatterlist *sg, |
291 | int nents, int dir) |
292 | { |
293 | struct scatterlist *s; |
294 | int i; |
295 | |
296 | #ifdef CONFIG_IOMMU_DEBUG |
297 | pr_debug("dma_map_sg overflow\n" ); |
298 | #endif |
299 | |
300 | for_each_sg(sg, s, nents, i) { |
301 | unsigned long addr = sg_phys(sg: s); |
302 | |
303 | if (nonforced_iommu(dev, addr, size: s->length)) { |
304 | addr = dma_map_area(dev, phys_mem: addr, size: s->length, dir, align_mask: 0); |
305 | if (addr == DMA_MAPPING_ERROR) { |
306 | if (i > 0) |
307 | gart_unmap_sg(dev, sg, nents: i, dir, attrs: 0); |
308 | nents = 0; |
309 | sg[0].dma_length = 0; |
310 | break; |
311 | } |
312 | } |
313 | s->dma_address = addr; |
314 | s->dma_length = s->length; |
315 | } |
316 | flush_gart(); |
317 | |
318 | return nents; |
319 | } |
320 | |
321 | /* Map multiple scatterlist entries continuous into the first. */ |
322 | static int __dma_map_cont(struct device *dev, struct scatterlist *start, |
323 | int nelems, struct scatterlist *sout, |
324 | unsigned long pages) |
325 | { |
326 | unsigned long iommu_start = alloc_iommu(dev, size: pages, align_mask: 0); |
327 | unsigned long iommu_page = iommu_start; |
328 | struct scatterlist *s; |
329 | int i; |
330 | |
331 | if (iommu_start == -1) |
332 | return -ENOMEM; |
333 | |
334 | for_each_sg(start, s, nelems, i) { |
335 | unsigned long pages, addr; |
336 | unsigned long phys_addr = s->dma_address; |
337 | |
338 | BUG_ON(s != start && s->offset); |
339 | if (s == start) { |
340 | sout->dma_address = iommu_bus_base; |
341 | sout->dma_address += iommu_page*PAGE_SIZE + s->offset; |
342 | sout->dma_length = s->length; |
343 | } else { |
344 | sout->dma_length += s->length; |
345 | } |
346 | |
347 | addr = phys_addr; |
348 | pages = iommu_num_pages(addr: s->offset, len: s->length, PAGE_SIZE); |
349 | while (pages--) { |
350 | iommu_gatt_base[iommu_page] = GPTE_ENCODE(addr); |
351 | addr += PAGE_SIZE; |
352 | iommu_page++; |
353 | } |
354 | } |
355 | BUG_ON(iommu_page - iommu_start != pages); |
356 | |
357 | return 0; |
358 | } |
359 | |
360 | static inline int |
361 | dma_map_cont(struct device *dev, struct scatterlist *start, int nelems, |
362 | struct scatterlist *sout, unsigned long pages, int need) |
363 | { |
364 | if (!need) { |
365 | BUG_ON(nelems != 1); |
366 | sout->dma_address = start->dma_address; |
367 | sout->dma_length = start->length; |
368 | return 0; |
369 | } |
370 | return __dma_map_cont(dev, start, nelems, sout, pages); |
371 | } |
372 | |
373 | /* |
374 | * DMA map all entries in a scatterlist. |
375 | * Merge chunks that have page aligned sizes into a continuous mapping. |
376 | */ |
377 | static int gart_map_sg(struct device *dev, struct scatterlist *sg, int nents, |
378 | enum dma_data_direction dir, unsigned long attrs) |
379 | { |
380 | struct scatterlist *s, *ps, *start_sg, *sgmap; |
381 | int need = 0, nextneed, i, out, start, ret; |
382 | unsigned long pages = 0; |
383 | unsigned int seg_size; |
384 | unsigned int max_seg_size; |
385 | |
386 | if (nents == 0) |
387 | return -EINVAL; |
388 | |
389 | out = 0; |
390 | start = 0; |
391 | start_sg = sg; |
392 | sgmap = sg; |
393 | seg_size = 0; |
394 | max_seg_size = dma_get_max_seg_size(dev); |
395 | ps = NULL; /* shut up gcc */ |
396 | |
397 | for_each_sg(sg, s, nents, i) { |
398 | dma_addr_t addr = sg_phys(sg: s); |
399 | |
400 | s->dma_address = addr; |
401 | BUG_ON(s->length == 0); |
402 | |
403 | nextneed = need_iommu(dev, addr, size: s->length); |
404 | |
405 | /* Handle the previous not yet processed entries */ |
406 | if (i > start) { |
407 | /* |
408 | * Can only merge when the last chunk ends on a |
409 | * page boundary and the new one doesn't have an |
410 | * offset. |
411 | */ |
412 | if (!iommu_merge || !nextneed || !need || s->offset || |
413 | (s->length + seg_size > max_seg_size) || |
414 | (ps->offset + ps->length) % PAGE_SIZE) { |
415 | ret = dma_map_cont(dev, start: start_sg, nelems: i - start, |
416 | sout: sgmap, pages, need); |
417 | if (ret < 0) |
418 | goto error; |
419 | out++; |
420 | |
421 | seg_size = 0; |
422 | sgmap = sg_next(sgmap); |
423 | pages = 0; |
424 | start = i; |
425 | start_sg = s; |
426 | } |
427 | } |
428 | |
429 | seg_size += s->length; |
430 | need = nextneed; |
431 | pages += iommu_num_pages(addr: s->offset, len: s->length, PAGE_SIZE); |
432 | ps = s; |
433 | } |
434 | ret = dma_map_cont(dev, start: start_sg, nelems: i - start, sout: sgmap, pages, need); |
435 | if (ret < 0) |
436 | goto error; |
437 | out++; |
438 | flush_gart(); |
439 | if (out < nents) { |
440 | sgmap = sg_next(sgmap); |
441 | sgmap->dma_length = 0; |
442 | } |
443 | return out; |
444 | |
445 | error: |
446 | flush_gart(); |
447 | gart_unmap_sg(dev, sg, nents: out, dir, attrs: 0); |
448 | |
449 | /* When it was forced or merged try again in a dumb way */ |
450 | if (force_iommu || iommu_merge) { |
451 | out = dma_map_sg_nonforce(dev, sg, nents, dir); |
452 | if (out > 0) |
453 | return out; |
454 | } |
455 | if (panic_on_overflow) |
456 | panic(fmt: "dma_map_sg: overflow on %lu pages\n" , pages); |
457 | |
458 | iommu_full(dev, size: pages << PAGE_SHIFT, dir); |
459 | return ret; |
460 | } |
461 | |
462 | /* allocate and map a coherent mapping */ |
463 | static void * |
464 | gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr, |
465 | gfp_t flag, unsigned long attrs) |
466 | { |
467 | void *vaddr; |
468 | |
469 | vaddr = dma_direct_alloc(dev, size, dma_handle: dma_addr, gfp: flag, attrs); |
470 | if (!vaddr || |
471 | !force_iommu || dev->coherent_dma_mask <= DMA_BIT_MASK(24)) |
472 | return vaddr; |
473 | |
474 | *dma_addr = dma_map_area(dev, virt_to_phys(address: vaddr), size, |
475 | dir: DMA_BIDIRECTIONAL, align_mask: (1UL << get_order(size)) - 1); |
476 | flush_gart(); |
477 | if (unlikely(*dma_addr == DMA_MAPPING_ERROR)) |
478 | goto out_free; |
479 | return vaddr; |
480 | out_free: |
481 | dma_direct_free(dev, size, cpu_addr: vaddr, dma_addr: *dma_addr, attrs); |
482 | return NULL; |
483 | } |
484 | |
485 | /* free a coherent mapping */ |
486 | static void |
487 | gart_free_coherent(struct device *dev, size_t size, void *vaddr, |
488 | dma_addr_t dma_addr, unsigned long attrs) |
489 | { |
490 | gart_unmap_page(dev, dma_addr, size, dir: DMA_BIDIRECTIONAL, attrs: 0); |
491 | dma_direct_free(dev, size, cpu_addr: vaddr, dma_addr, attrs); |
492 | } |
493 | |
494 | static int no_agp; |
495 | |
496 | static __init unsigned long check_iommu_size(unsigned long aper, u64 aper_size) |
497 | { |
498 | unsigned long a; |
499 | |
500 | if (!iommu_size) { |
501 | iommu_size = aper_size; |
502 | if (!no_agp) |
503 | iommu_size /= 2; |
504 | } |
505 | |
506 | a = aper + iommu_size; |
507 | iommu_size -= round_up(a, PMD_SIZE) - a; |
508 | |
509 | if (iommu_size < 64*1024*1024) { |
510 | pr_warn("PCI-DMA: Warning: Small IOMMU %luMB." |
511 | " Consider increasing the AGP aperture in BIOS\n" , |
512 | iommu_size >> 20); |
513 | } |
514 | |
515 | return iommu_size; |
516 | } |
517 | |
518 | static __init unsigned read_aperture(struct pci_dev *dev, u32 *size) |
519 | { |
520 | unsigned aper_size = 0, aper_base_32, aper_order; |
521 | u64 aper_base; |
522 | |
523 | pci_read_config_dword(dev, AMD64_GARTAPERTUREBASE, val: &aper_base_32); |
524 | pci_read_config_dword(dev, AMD64_GARTAPERTURECTL, val: &aper_order); |
525 | aper_order = (aper_order >> 1) & 7; |
526 | |
527 | aper_base = aper_base_32 & 0x7fff; |
528 | aper_base <<= 25; |
529 | |
530 | aper_size = (32 * 1024 * 1024) << aper_order; |
531 | if (aper_base + aper_size > 0x100000000UL || !aper_size) |
532 | aper_base = 0; |
533 | |
534 | *size = aper_size; |
535 | return aper_base; |
536 | } |
537 | |
538 | static void enable_gart_translations(void) |
539 | { |
540 | int i; |
541 | |
542 | if (!amd_nb_has_feature(AMD_NB_GART)) |
543 | return; |
544 | |
545 | for (i = 0; i < amd_nb_num(); i++) { |
546 | struct pci_dev *dev = node_to_amd_nb(node: i)->misc; |
547 | |
548 | enable_gart_translation(dev, __pa(agp_gatt_table)); |
549 | } |
550 | |
551 | /* Flush the GART-TLB to remove stale entries */ |
552 | amd_flush_garts(); |
553 | } |
554 | |
555 | /* |
556 | * If fix_up_north_bridges is set, the north bridges have to be fixed up on |
557 | * resume in the same way as they are handled in gart_iommu_hole_init(). |
558 | */ |
559 | static bool fix_up_north_bridges; |
560 | static u32 aperture_order; |
561 | static u32 aperture_alloc; |
562 | |
563 | void set_up_gart_resume(u32 aper_order, u32 aper_alloc) |
564 | { |
565 | fix_up_north_bridges = true; |
566 | aperture_order = aper_order; |
567 | aperture_alloc = aper_alloc; |
568 | } |
569 | |
570 | static void gart_fixup_northbridges(void) |
571 | { |
572 | int i; |
573 | |
574 | if (!fix_up_north_bridges) |
575 | return; |
576 | |
577 | if (!amd_nb_has_feature(AMD_NB_GART)) |
578 | return; |
579 | |
580 | pr_info("PCI-DMA: Restoring GART aperture settings\n" ); |
581 | |
582 | for (i = 0; i < amd_nb_num(); i++) { |
583 | struct pci_dev *dev = node_to_amd_nb(node: i)->misc; |
584 | |
585 | /* |
586 | * Don't enable translations just yet. That is the next |
587 | * step. Restore the pre-suspend aperture settings. |
588 | */ |
589 | gart_set_size_and_enable(dev, order: aperture_order); |
590 | pci_write_config_dword(dev, AMD64_GARTAPERTUREBASE, val: aperture_alloc >> 25); |
591 | } |
592 | } |
593 | |
594 | static void gart_resume(void) |
595 | { |
596 | pr_info("PCI-DMA: Resuming GART IOMMU\n" ); |
597 | |
598 | gart_fixup_northbridges(); |
599 | |
600 | enable_gart_translations(); |
601 | } |
602 | |
603 | static struct syscore_ops gart_syscore_ops = { |
604 | .resume = gart_resume, |
605 | |
606 | }; |
607 | |
608 | /* |
609 | * Private Northbridge GATT initialization in case we cannot use the |
610 | * AGP driver for some reason. |
611 | */ |
612 | static __init int init_amd_gatt(struct agp_kern_info *info) |
613 | { |
614 | unsigned aper_size, gatt_size, new_aper_size; |
615 | unsigned aper_base, new_aper_base; |
616 | struct pci_dev *dev; |
617 | void *gatt; |
618 | int i; |
619 | |
620 | pr_info("PCI-DMA: Disabling AGP.\n" ); |
621 | |
622 | aper_size = aper_base = info->aper_size = 0; |
623 | dev = NULL; |
624 | for (i = 0; i < amd_nb_num(); i++) { |
625 | dev = node_to_amd_nb(node: i)->misc; |
626 | new_aper_base = read_aperture(dev, size: &new_aper_size); |
627 | if (!new_aper_base) |
628 | goto nommu; |
629 | |
630 | if (!aper_base) { |
631 | aper_size = new_aper_size; |
632 | aper_base = new_aper_base; |
633 | } |
634 | if (aper_size != new_aper_size || aper_base != new_aper_base) |
635 | goto nommu; |
636 | } |
637 | if (!aper_base) |
638 | goto nommu; |
639 | |
640 | info->aper_base = aper_base; |
641 | info->aper_size = aper_size >> 20; |
642 | |
643 | gatt_size = (aper_size >> PAGE_SHIFT) * sizeof(u32); |
644 | gatt = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, |
645 | order: get_order(size: gatt_size)); |
646 | if (!gatt) |
647 | panic(fmt: "Cannot allocate GATT table" ); |
648 | if (set_memory_uc(addr: (unsigned long)gatt, numpages: gatt_size >> PAGE_SHIFT)) |
649 | panic(fmt: "Could not set GART PTEs to uncacheable pages" ); |
650 | |
651 | agp_gatt_table = gatt; |
652 | |
653 | register_syscore_ops(ops: &gart_syscore_ops); |
654 | |
655 | flush_gart(); |
656 | |
657 | pr_info("PCI-DMA: aperture base @ %x size %u KB\n" , |
658 | aper_base, aper_size>>10); |
659 | |
660 | return 0; |
661 | |
662 | nommu: |
663 | /* Should not happen anymore */ |
664 | pr_warn("PCI-DMA: More than 4GB of RAM and no IOMMU - falling back to iommu=soft.\n" ); |
665 | return -1; |
666 | } |
667 | |
668 | static const struct dma_map_ops gart_dma_ops = { |
669 | .map_sg = gart_map_sg, |
670 | .unmap_sg = gart_unmap_sg, |
671 | .map_page = gart_map_page, |
672 | .unmap_page = gart_unmap_page, |
673 | .alloc = gart_alloc_coherent, |
674 | .free = gart_free_coherent, |
675 | .mmap = dma_common_mmap, |
676 | .get_sgtable = dma_common_get_sgtable, |
677 | .dma_supported = dma_direct_supported, |
678 | .get_required_mask = dma_direct_get_required_mask, |
679 | .alloc_pages = dma_direct_alloc_pages, |
680 | .free_pages = dma_direct_free_pages, |
681 | }; |
682 | |
683 | static void gart_iommu_shutdown(void) |
684 | { |
685 | struct pci_dev *dev; |
686 | int i; |
687 | |
688 | /* don't shutdown it if there is AGP installed */ |
689 | if (!no_agp) |
690 | return; |
691 | |
692 | if (!amd_nb_has_feature(AMD_NB_GART)) |
693 | return; |
694 | |
695 | for (i = 0; i < amd_nb_num(); i++) { |
696 | u32 ctl; |
697 | |
698 | dev = node_to_amd_nb(node: i)->misc; |
699 | pci_read_config_dword(dev, AMD64_GARTAPERTURECTL, val: &ctl); |
700 | |
701 | ctl &= ~GARTEN; |
702 | |
703 | pci_write_config_dword(dev, AMD64_GARTAPERTURECTL, val: ctl); |
704 | } |
705 | } |
706 | |
707 | int __init gart_iommu_init(void) |
708 | { |
709 | struct agp_kern_info info; |
710 | unsigned long iommu_start; |
711 | unsigned long aper_base, aper_size; |
712 | unsigned long start_pfn, end_pfn; |
713 | unsigned long scratch; |
714 | |
715 | if (!amd_nb_has_feature(AMD_NB_GART)) |
716 | return 0; |
717 | |
718 | #ifndef CONFIG_AGP_AMD64 |
719 | no_agp = 1; |
720 | #else |
721 | /* Makefile puts PCI initialization via subsys_initcall first. */ |
722 | /* Add other AMD AGP bridge drivers here */ |
723 | no_agp = no_agp || |
724 | (agp_amd64_init() < 0) || |
725 | (agp_copy_info(agp_bridge, &info) < 0); |
726 | #endif |
727 | |
728 | if (no_iommu || |
729 | (!force_iommu && max_pfn <= MAX_DMA32_PFN) || |
730 | !gart_iommu_aperture || |
731 | (no_agp && init_amd_gatt(info: &info) < 0)) { |
732 | if (max_pfn > MAX_DMA32_PFN) { |
733 | pr_warn("More than 4GB of memory but GART IOMMU not available.\n" ); |
734 | pr_warn("falling back to iommu=soft.\n" ); |
735 | } |
736 | return 0; |
737 | } |
738 | |
739 | /* need to map that range */ |
740 | aper_size = info.aper_size << 20; |
741 | aper_base = info.aper_base; |
742 | end_pfn = (aper_base>>PAGE_SHIFT) + (aper_size>>PAGE_SHIFT); |
743 | |
744 | start_pfn = PFN_DOWN(aper_base); |
745 | if (!pfn_range_is_mapped(start_pfn, end_pfn)) |
746 | init_memory_mapping(start: start_pfn<<PAGE_SHIFT, end: end_pfn<<PAGE_SHIFT, |
747 | PAGE_KERNEL); |
748 | |
749 | pr_info("PCI-DMA: using GART IOMMU.\n" ); |
750 | iommu_size = check_iommu_size(aper: info.aper_base, aper_size); |
751 | iommu_pages = iommu_size >> PAGE_SHIFT; |
752 | |
753 | iommu_gart_bitmap = (void *) __get_free_pages(GFP_KERNEL | __GFP_ZERO, |
754 | order: get_order(size: iommu_pages/8)); |
755 | if (!iommu_gart_bitmap) |
756 | panic(fmt: "Cannot allocate iommu bitmap\n" ); |
757 | |
758 | pr_info("PCI-DMA: Reserving %luMB of IOMMU area in the AGP aperture\n" , |
759 | iommu_size >> 20); |
760 | |
761 | agp_memory_reserved = iommu_size; |
762 | iommu_start = aper_size - iommu_size; |
763 | iommu_bus_base = info.aper_base + iommu_start; |
764 | iommu_gatt_base = agp_gatt_table + (iommu_start>>PAGE_SHIFT); |
765 | |
766 | /* |
767 | * Unmap the IOMMU part of the GART. The alias of the page is |
768 | * always mapped with cache enabled and there is no full cache |
769 | * coherency across the GART remapping. The unmapping avoids |
770 | * automatic prefetches from the CPU allocating cache lines in |
771 | * there. All CPU accesses are done via the direct mapping to |
772 | * the backing memory. The GART address is only used by PCI |
773 | * devices. |
774 | */ |
775 | set_memory_np(addr: (unsigned long)__va(iommu_bus_base), |
776 | numpages: iommu_size >> PAGE_SHIFT); |
777 | /* |
778 | * Tricky. The GART table remaps the physical memory range, |
779 | * so the CPU wont notice potential aliases and if the memory |
780 | * is remapped to UC later on, we might surprise the PCI devices |
781 | * with a stray writeout of a cacheline. So play it sure and |
782 | * do an explicit, full-scale wbinvd() _after_ having marked all |
783 | * the pages as Not-Present: |
784 | */ |
785 | wbinvd(); |
786 | |
787 | /* |
788 | * Now all caches are flushed and we can safely enable |
789 | * GART hardware. Doing it early leaves the possibility |
790 | * of stale cache entries that can lead to GART PTE |
791 | * errors. |
792 | */ |
793 | enable_gart_translations(); |
794 | |
795 | /* |
796 | * Try to workaround a bug (thanks to BenH): |
797 | * Set unmapped entries to a scratch page instead of 0. |
798 | * Any prefetches that hit unmapped entries won't get an bus abort |
799 | * then. (P2P bridge may be prefetching on DMA reads). |
800 | */ |
801 | scratch = get_zeroed_page(GFP_KERNEL); |
802 | if (!scratch) |
803 | panic(fmt: "Cannot allocate iommu scratch page" ); |
804 | gart_unmapped_entry = GPTE_ENCODE(__pa(scratch)); |
805 | |
806 | flush_gart(); |
807 | dma_ops = &gart_dma_ops; |
808 | x86_platform.iommu_shutdown = gart_iommu_shutdown; |
809 | x86_swiotlb_enable = false; |
810 | |
811 | return 0; |
812 | } |
813 | |
814 | void __init gart_parse_options(char *p) |
815 | { |
816 | int arg; |
817 | |
818 | if (isdigit(c: *p) && get_option(str: &p, pint: &arg)) |
819 | iommu_size = arg; |
820 | if (!strncmp(p, "fullflush" , 9)) |
821 | iommu_fullflush = 1; |
822 | if (!strncmp(p, "nofullflush" , 11)) |
823 | iommu_fullflush = 0; |
824 | if (!strncmp(p, "noagp" , 5)) |
825 | no_agp = 1; |
826 | if (!strncmp(p, "noaperture" , 10)) |
827 | fix_aperture = 0; |
828 | /* duplicated from pci-dma.c */ |
829 | if (!strncmp(p, "force" , 5)) |
830 | gart_iommu_aperture_allowed = 1; |
831 | if (!strncmp(p, "allowed" , 7)) |
832 | gart_iommu_aperture_allowed = 1; |
833 | if (!strncmp(p, "memaper" , 7)) { |
834 | fallback_aper_force = 1; |
835 | p += 7; |
836 | if (*p == '=') { |
837 | ++p; |
838 | if (get_option(str: &p, pint: &arg)) |
839 | fallback_aper_order = arg; |
840 | } |
841 | } |
842 | } |
843 | |