1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Firmware replacement code. |
4 | * |
5 | * Work around broken BIOSes that don't set an aperture, only set the |
6 | * aperture in the AGP bridge, or set too small aperture. |
7 | * |
8 | * If all fails map the aperture over some low memory. This is cheaper than |
9 | * doing bounce buffering. The memory is lost. This is done at early boot |
10 | * because only the bootmem allocator can allocate 32+MB. |
11 | * |
12 | * Copyright 2002 Andi Kleen, SuSE Labs. |
13 | */ |
14 | #define pr_fmt(fmt) "AGP: " fmt |
15 | |
16 | #include <linux/kernel.h> |
17 | #include <linux/kcore.h> |
18 | #include <linux/types.h> |
19 | #include <linux/init.h> |
20 | #include <linux/memblock.h> |
21 | #include <linux/mmzone.h> |
22 | #include <linux/pci_ids.h> |
23 | #include <linux/pci.h> |
24 | #include <linux/bitops.h> |
25 | #include <linux/suspend.h> |
26 | #include <asm/e820/api.h> |
27 | #include <asm/io.h> |
28 | #include <asm/iommu.h> |
29 | #include <asm/gart.h> |
30 | #include <asm/pci-direct.h> |
31 | #include <asm/dma.h> |
32 | #include <asm/amd_nb.h> |
33 | #include <asm/x86_init.h> |
34 | #include <linux/crash_dump.h> |
35 | |
36 | /* |
37 | * Using 512M as goal, in case kexec will load kernel_big |
38 | * that will do the on-position decompress, and could overlap with |
39 | * with the gart aperture that is used. |
40 | * Sequence: |
41 | * kernel_small |
42 | * ==> kexec (with kdump trigger path or gart still enabled) |
43 | * ==> kernel_small (gart area become e820_reserved) |
44 | * ==> kexec (with kdump trigger path or gart still enabled) |
45 | * ==> kerne_big (uncompressed size will be big than 64M or 128M) |
46 | * So don't use 512M below as gart iommu, leave the space for kernel |
47 | * code for safe. |
48 | */ |
49 | #define GART_MIN_ADDR (512ULL << 20) |
50 | #define GART_MAX_ADDR (1ULL << 32) |
51 | |
52 | int gart_iommu_aperture; |
53 | int gart_iommu_aperture_disabled __initdata; |
54 | int gart_iommu_aperture_allowed __initdata; |
55 | |
56 | int fallback_aper_order __initdata = 1; /* 64MB */ |
57 | int fallback_aper_force __initdata; |
58 | |
59 | int fix_aperture __initdata = 1; |
60 | |
61 | #if defined(CONFIG_PROC_VMCORE) || defined(CONFIG_PROC_KCORE) |
62 | /* |
63 | * If the first kernel maps the aperture over e820 RAM, the kdump kernel will |
64 | * use the same range because it will remain configured in the northbridge. |
65 | * Trying to dump this area via /proc/vmcore may crash the machine, so exclude |
66 | * it from vmcore. |
67 | */ |
68 | static unsigned long aperture_pfn_start, aperture_page_count; |
69 | |
70 | static int gart_mem_pfn_is_ram(unsigned long pfn) |
71 | { |
72 | return likely((pfn < aperture_pfn_start) || |
73 | (pfn >= aperture_pfn_start + aperture_page_count)); |
74 | } |
75 | |
76 | static void __init exclude_from_core(u64 aper_base, u32 aper_order) |
77 | { |
78 | aperture_pfn_start = aper_base >> PAGE_SHIFT; |
79 | aperture_page_count = (32 * 1024 * 1024) << aper_order >> PAGE_SHIFT; |
80 | #ifdef CONFIG_PROC_VMCORE |
81 | WARN_ON(register_oldmem_pfn_is_ram(&gart_mem_pfn_is_ram)); |
82 | #endif |
83 | #ifdef CONFIG_PROC_KCORE |
84 | WARN_ON(register_mem_pfn_is_ram(&gart_mem_pfn_is_ram)); |
85 | #endif |
86 | } |
87 | #else |
88 | static void exclude_from_core(u64 aper_base, u32 aper_order) |
89 | { |
90 | } |
91 | #endif |
92 | |
93 | /* This code runs before the PCI subsystem is initialized, so just |
94 | access the northbridge directly. */ |
95 | |
96 | static u32 __init allocate_aperture(void) |
97 | { |
98 | u32 aper_size; |
99 | unsigned long addr; |
100 | |
101 | /* aper_size should <= 1G */ |
102 | if (fallback_aper_order > 5) |
103 | fallback_aper_order = 5; |
104 | aper_size = (32 * 1024 * 1024) << fallback_aper_order; |
105 | |
106 | /* |
107 | * Aperture has to be naturally aligned. This means a 2GB aperture |
108 | * won't have much chance of finding a place in the lower 4GB of |
109 | * memory. Unfortunately we cannot move it up because that would |
110 | * make the IOMMU useless. |
111 | */ |
112 | addr = memblock_find_in_range(GART_MIN_ADDR, GART_MAX_ADDR, |
113 | aper_size, aper_size); |
114 | if (!addr) { |
115 | pr_err("Cannot allocate aperture memory hole [mem %#010lx-%#010lx] (%uKB)\n" , |
116 | addr, addr + aper_size - 1, aper_size >> 10); |
117 | return 0; |
118 | } |
119 | memblock_reserve(addr, aper_size); |
120 | pr_info("Mapping aperture over RAM [mem %#010lx-%#010lx] (%uKB)\n" , |
121 | addr, addr + aper_size - 1, aper_size >> 10); |
122 | register_nosave_region(addr >> PAGE_SHIFT, |
123 | (addr+aper_size) >> PAGE_SHIFT); |
124 | |
125 | return (u32)addr; |
126 | } |
127 | |
128 | |
129 | /* Find a PCI capability */ |
130 | static u32 __init find_cap(int bus, int slot, int func, int cap) |
131 | { |
132 | int bytes; |
133 | u8 pos; |
134 | |
135 | if (!(read_pci_config_16(bus, slot, func, PCI_STATUS) & |
136 | PCI_STATUS_CAP_LIST)) |
137 | return 0; |
138 | |
139 | pos = read_pci_config_byte(bus, slot, func, PCI_CAPABILITY_LIST); |
140 | for (bytes = 0; bytes < 48 && pos >= 0x40; bytes++) { |
141 | u8 id; |
142 | |
143 | pos &= ~3; |
144 | id = read_pci_config_byte(bus, slot, func, pos+PCI_CAP_LIST_ID); |
145 | if (id == 0xff) |
146 | break; |
147 | if (id == cap) |
148 | return pos; |
149 | pos = read_pci_config_byte(bus, slot, func, |
150 | pos+PCI_CAP_LIST_NEXT); |
151 | } |
152 | return 0; |
153 | } |
154 | |
155 | /* Read a standard AGPv3 bridge header */ |
156 | static u32 __init read_agp(int bus, int slot, int func, int cap, u32 *order) |
157 | { |
158 | u32 apsize; |
159 | u32 apsizereg; |
160 | int nbits; |
161 | u32 aper_low, aper_hi; |
162 | u64 aper; |
163 | u32 old_order; |
164 | |
165 | pr_info("pci 0000:%02x:%02x:%02x: AGP bridge\n" , bus, slot, func); |
166 | apsizereg = read_pci_config_16(bus, slot, func, cap + 0x14); |
167 | if (apsizereg == 0xffffffff) { |
168 | pr_err("pci 0000:%02x:%02x.%d: APSIZE unreadable\n" , |
169 | bus, slot, func); |
170 | return 0; |
171 | } |
172 | |
173 | /* old_order could be the value from NB gart setting */ |
174 | old_order = *order; |
175 | |
176 | apsize = apsizereg & 0xfff; |
177 | /* Some BIOS use weird encodings not in the AGPv3 table. */ |
178 | if (apsize & 0xff) |
179 | apsize |= 0xf00; |
180 | nbits = hweight16(apsize); |
181 | *order = 7 - nbits; |
182 | if ((int)*order < 0) /* < 32MB */ |
183 | *order = 0; |
184 | |
185 | aper_low = read_pci_config(bus, slot, func, 0x10); |
186 | aper_hi = read_pci_config(bus, slot, func, 0x14); |
187 | aper = (aper_low & ~((1<<22)-1)) | ((u64)aper_hi << 32); |
188 | |
189 | /* |
190 | * On some sick chips, APSIZE is 0. It means it wants 4G |
191 | * so let double check that order, and lets trust AMD NB settings: |
192 | */ |
193 | pr_info("pci 0000:%02x:%02x.%d: AGP aperture [bus addr %#010Lx-%#010Lx] (old size %uMB)\n" , |
194 | bus, slot, func, aper, aper + (32ULL << (old_order + 20)) - 1, |
195 | 32 << old_order); |
196 | if (aper + (32ULL<<(20 + *order)) > 0x100000000ULL) { |
197 | pr_info("pci 0000:%02x:%02x.%d: AGP aperture size %uMB (APSIZE %#x) is not right, using settings from NB\n" , |
198 | bus, slot, func, 32 << *order, apsizereg); |
199 | *order = old_order; |
200 | } |
201 | |
202 | pr_info("pci 0000:%02x:%02x.%d: AGP aperture [bus addr %#010Lx-%#010Lx] (%uMB, APSIZE %#x)\n" , |
203 | bus, slot, func, aper, aper + (32ULL << (*order + 20)) - 1, |
204 | 32 << *order, apsizereg); |
205 | |
206 | if (!aperture_valid(aper, (32*1024*1024) << *order, 32<<20)) |
207 | return 0; |
208 | return (u32)aper; |
209 | } |
210 | |
211 | /* |
212 | * Look for an AGP bridge. Windows only expects the aperture in the |
213 | * AGP bridge and some BIOS forget to initialize the Northbridge too. |
214 | * Work around this here. |
215 | * |
216 | * Do an PCI bus scan by hand because we're running before the PCI |
217 | * subsystem. |
218 | * |
219 | * All AMD AGP bridges are AGPv3 compliant, so we can do this scan |
220 | * generically. It's probably overkill to always scan all slots because |
221 | * the AGP bridges should be always an own bus on the HT hierarchy, |
222 | * but do it here for future safety. |
223 | */ |
224 | static u32 __init search_agp_bridge(u32 *order, int *valid_agp) |
225 | { |
226 | int bus, slot, func; |
227 | |
228 | /* Poor man's PCI discovery */ |
229 | for (bus = 0; bus < 256; bus++) { |
230 | for (slot = 0; slot < 32; slot++) { |
231 | for (func = 0; func < 8; func++) { |
232 | u32 class, cap; |
233 | u8 type; |
234 | class = read_pci_config(bus, slot, func, |
235 | PCI_CLASS_REVISION); |
236 | if (class == 0xffffffff) |
237 | break; |
238 | |
239 | switch (class >> 16) { |
240 | case PCI_CLASS_BRIDGE_HOST: |
241 | case PCI_CLASS_BRIDGE_OTHER: /* needed? */ |
242 | /* AGP bridge? */ |
243 | cap = find_cap(bus, slot, func, |
244 | PCI_CAP_ID_AGP); |
245 | if (!cap) |
246 | break; |
247 | *valid_agp = 1; |
248 | return read_agp(bus, slot, func, cap, |
249 | order); |
250 | } |
251 | |
252 | /* No multi-function device? */ |
253 | type = read_pci_config_byte(bus, slot, func, |
254 | PCI_HEADER_TYPE); |
255 | if (!(type & 0x80)) |
256 | break; |
257 | } |
258 | } |
259 | } |
260 | pr_info("No AGP bridge found\n" ); |
261 | |
262 | return 0; |
263 | } |
264 | |
265 | static bool gart_fix_e820 __initdata = true; |
266 | |
267 | static int __init parse_gart_mem(char *p) |
268 | { |
269 | return kstrtobool(p, &gart_fix_e820); |
270 | } |
271 | early_param("gart_fix_e820" , parse_gart_mem); |
272 | |
273 | /* |
274 | * With kexec/kdump, if the first kernel doesn't shut down the GART and the |
275 | * second kernel allocates a different GART region, there might be two |
276 | * overlapping GART regions present: |
277 | * |
278 | * - the first still used by the GART initialized in the first kernel. |
279 | * - (sub-)set of it used as normal RAM by the second kernel. |
280 | * |
281 | * which leads to memory corruptions and a kernel panic eventually. |
282 | * |
283 | * This can also happen if the BIOS has forgotten to mark the GART region |
284 | * as reserved. |
285 | * |
286 | * Try to update the e820 map to mark that new region as reserved. |
287 | */ |
288 | void __init early_gart_iommu_check(void) |
289 | { |
290 | u32 agp_aper_order = 0; |
291 | int i, fix, slot, valid_agp = 0; |
292 | u32 ctl; |
293 | u32 aper_size = 0, aper_order = 0, last_aper_order = 0; |
294 | u64 aper_base = 0, last_aper_base = 0; |
295 | int aper_enabled = 0, last_aper_enabled = 0, last_valid = 0; |
296 | |
297 | if (!amd_gart_present()) |
298 | return; |
299 | |
300 | if (!early_pci_allowed()) |
301 | return; |
302 | |
303 | /* This is mostly duplicate of iommu_hole_init */ |
304 | search_agp_bridge(&agp_aper_order, &valid_agp); |
305 | |
306 | fix = 0; |
307 | for (i = 0; amd_nb_bus_dev_ranges[i].dev_limit; i++) { |
308 | int bus; |
309 | int dev_base, dev_limit; |
310 | |
311 | bus = amd_nb_bus_dev_ranges[i].bus; |
312 | dev_base = amd_nb_bus_dev_ranges[i].dev_base; |
313 | dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; |
314 | |
315 | for (slot = dev_base; slot < dev_limit; slot++) { |
316 | if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) |
317 | continue; |
318 | |
319 | ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); |
320 | aper_enabled = ctl & GARTEN; |
321 | aper_order = (ctl >> 1) & 7; |
322 | aper_size = (32 * 1024 * 1024) << aper_order; |
323 | aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; |
324 | aper_base <<= 25; |
325 | |
326 | if (last_valid) { |
327 | if ((aper_order != last_aper_order) || |
328 | (aper_base != last_aper_base) || |
329 | (aper_enabled != last_aper_enabled)) { |
330 | fix = 1; |
331 | break; |
332 | } |
333 | } |
334 | |
335 | last_aper_order = aper_order; |
336 | last_aper_base = aper_base; |
337 | last_aper_enabled = aper_enabled; |
338 | last_valid = 1; |
339 | } |
340 | } |
341 | |
342 | if (!fix && !aper_enabled) |
343 | return; |
344 | |
345 | if (!aper_base || !aper_size || aper_base + aper_size > 0x100000000UL) |
346 | fix = 1; |
347 | |
348 | if (gart_fix_e820 && !fix && aper_enabled) { |
349 | if (e820__mapped_any(aper_base, aper_base + aper_size, |
350 | E820_TYPE_RAM)) { |
351 | /* reserve it, so we can reuse it in second kernel */ |
352 | pr_info("e820: reserve [mem %#010Lx-%#010Lx] for GART\n" , |
353 | aper_base, aper_base + aper_size - 1); |
354 | e820__range_add(aper_base, aper_size, E820_TYPE_RESERVED); |
355 | e820__update_table_print(); |
356 | } |
357 | } |
358 | |
359 | if (valid_agp) |
360 | return; |
361 | |
362 | /* disable them all at first */ |
363 | for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { |
364 | int bus; |
365 | int dev_base, dev_limit; |
366 | |
367 | bus = amd_nb_bus_dev_ranges[i].bus; |
368 | dev_base = amd_nb_bus_dev_ranges[i].dev_base; |
369 | dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; |
370 | |
371 | for (slot = dev_base; slot < dev_limit; slot++) { |
372 | if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) |
373 | continue; |
374 | |
375 | ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); |
376 | ctl &= ~GARTEN; |
377 | write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); |
378 | } |
379 | } |
380 | |
381 | } |
382 | |
383 | static int __initdata printed_gart_size_msg; |
384 | |
385 | int __init gart_iommu_hole_init(void) |
386 | { |
387 | u32 agp_aper_base = 0, agp_aper_order = 0; |
388 | u32 aper_size, aper_alloc = 0, aper_order = 0, last_aper_order = 0; |
389 | u64 aper_base, last_aper_base = 0; |
390 | int fix, slot, valid_agp = 0; |
391 | int i, node; |
392 | |
393 | if (!amd_gart_present()) |
394 | return -ENODEV; |
395 | |
396 | if (gart_iommu_aperture_disabled || !fix_aperture || |
397 | !early_pci_allowed()) |
398 | return -ENODEV; |
399 | |
400 | pr_info("Checking aperture...\n" ); |
401 | |
402 | if (!fallback_aper_force) |
403 | agp_aper_base = search_agp_bridge(&agp_aper_order, &valid_agp); |
404 | |
405 | fix = 0; |
406 | node = 0; |
407 | for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { |
408 | int bus; |
409 | int dev_base, dev_limit; |
410 | u32 ctl; |
411 | |
412 | bus = amd_nb_bus_dev_ranges[i].bus; |
413 | dev_base = amd_nb_bus_dev_ranges[i].dev_base; |
414 | dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; |
415 | |
416 | for (slot = dev_base; slot < dev_limit; slot++) { |
417 | if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) |
418 | continue; |
419 | |
420 | iommu_detected = 1; |
421 | gart_iommu_aperture = 1; |
422 | x86_init.iommu.iommu_init = gart_iommu_init; |
423 | |
424 | ctl = read_pci_config(bus, slot, 3, |
425 | AMD64_GARTAPERTURECTL); |
426 | |
427 | /* |
428 | * Before we do anything else disable the GART. It may |
429 | * still be enabled if we boot into a crash-kernel here. |
430 | * Reconfiguring the GART while it is enabled could have |
431 | * unknown side-effects. |
432 | */ |
433 | ctl &= ~GARTEN; |
434 | write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); |
435 | |
436 | aper_order = (ctl >> 1) & 7; |
437 | aper_size = (32 * 1024 * 1024) << aper_order; |
438 | aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; |
439 | aper_base <<= 25; |
440 | |
441 | pr_info("Node %d: aperture [bus addr %#010Lx-%#010Lx] (%uMB)\n" , |
442 | node, aper_base, aper_base + aper_size - 1, |
443 | aper_size >> 20); |
444 | node++; |
445 | |
446 | if (!aperture_valid(aper_base, aper_size, 64<<20)) { |
447 | if (valid_agp && agp_aper_base && |
448 | agp_aper_base == aper_base && |
449 | agp_aper_order == aper_order) { |
450 | /* the same between two setting from NB and agp */ |
451 | if (!no_iommu && |
452 | max_pfn > MAX_DMA32_PFN && |
453 | !printed_gart_size_msg) { |
454 | pr_err("you are using iommu with agp, but GART size is less than 64MB\n" ); |
455 | pr_err("please increase GART size in your BIOS setup\n" ); |
456 | pr_err("if BIOS doesn't have that option, contact your HW vendor!\n" ); |
457 | printed_gart_size_msg = 1; |
458 | } |
459 | } else { |
460 | fix = 1; |
461 | goto out; |
462 | } |
463 | } |
464 | |
465 | if ((last_aper_order && aper_order != last_aper_order) || |
466 | (last_aper_base && aper_base != last_aper_base)) { |
467 | fix = 1; |
468 | goto out; |
469 | } |
470 | last_aper_order = aper_order; |
471 | last_aper_base = aper_base; |
472 | } |
473 | } |
474 | |
475 | out: |
476 | if (!fix && !fallback_aper_force) { |
477 | if (last_aper_base) { |
478 | /* |
479 | * If this is the kdump kernel, the first kernel |
480 | * may have allocated the range over its e820 RAM |
481 | * and fixed up the northbridge |
482 | */ |
483 | exclude_from_core(last_aper_base, last_aper_order); |
484 | |
485 | return 1; |
486 | } |
487 | return 0; |
488 | } |
489 | |
490 | if (!fallback_aper_force) { |
491 | aper_alloc = agp_aper_base; |
492 | aper_order = agp_aper_order; |
493 | } |
494 | |
495 | if (aper_alloc) { |
496 | /* Got the aperture from the AGP bridge */ |
497 | } else if ((!no_iommu && max_pfn > MAX_DMA32_PFN) || |
498 | force_iommu || |
499 | valid_agp || |
500 | fallback_aper_force) { |
501 | pr_info("Your BIOS doesn't leave an aperture memory hole\n" ); |
502 | pr_info("Please enable the IOMMU option in the BIOS setup\n" ); |
503 | pr_info("This costs you %dMB of RAM\n" , |
504 | 32 << fallback_aper_order); |
505 | |
506 | aper_order = fallback_aper_order; |
507 | aper_alloc = allocate_aperture(); |
508 | if (!aper_alloc) { |
509 | /* |
510 | * Could disable AGP and IOMMU here, but it's |
511 | * probably not worth it. But the later users |
512 | * cannot deal with bad apertures and turning |
513 | * on the aperture over memory causes very |
514 | * strange problems, so it's better to panic |
515 | * early. |
516 | */ |
517 | panic("Not enough memory for aperture" ); |
518 | } |
519 | } else { |
520 | return 0; |
521 | } |
522 | |
523 | /* |
524 | * If this is the kdump kernel _and_ the first kernel did not |
525 | * configure the aperture in the northbridge, this range may |
526 | * overlap with the first kernel's memory. We can't access the |
527 | * range through vmcore even though it should be part of the dump. |
528 | */ |
529 | exclude_from_core(aper_alloc, aper_order); |
530 | |
531 | /* Fix up the north bridges */ |
532 | for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { |
533 | int bus, dev_base, dev_limit; |
534 | |
535 | /* |
536 | * Don't enable translation yet but enable GART IO and CPU |
537 | * accesses and set DISTLBWALKPRB since GART table memory is UC. |
538 | */ |
539 | u32 ctl = aper_order << 1; |
540 | |
541 | bus = amd_nb_bus_dev_ranges[i].bus; |
542 | dev_base = amd_nb_bus_dev_ranges[i].dev_base; |
543 | dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; |
544 | for (slot = dev_base; slot < dev_limit; slot++) { |
545 | if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) |
546 | continue; |
547 | |
548 | write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); |
549 | write_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE, aper_alloc >> 25); |
550 | } |
551 | } |
552 | |
553 | set_up_gart_resume(aper_order, aper_alloc); |
554 | |
555 | return 1; |
556 | } |
557 | |