1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Re-map IO memory to kernel address space so that we can access it. |
4 | * This is needed for high PCI addresses that aren't mapped in the |
5 | * 640k-1MB IO memory area on PC's |
6 | * |
7 | * (C) Copyright 1995 1996 Linus Torvalds |
8 | */ |
9 | |
10 | #include <linux/memblock.h> |
11 | #include <linux/init.h> |
12 | #include <linux/io.h> |
13 | #include <linux/ioport.h> |
14 | #include <linux/slab.h> |
15 | #include <linux/vmalloc.h> |
16 | #include <linux/mmiotrace.h> |
17 | #include <linux/cc_platform.h> |
18 | #include <linux/efi.h> |
19 | #include <linux/pgtable.h> |
20 | #include <linux/kmsan.h> |
21 | |
22 | #include <asm/set_memory.h> |
23 | #include <asm/e820/api.h> |
24 | #include <asm/efi.h> |
25 | #include <asm/fixmap.h> |
26 | #include <asm/tlbflush.h> |
27 | #include <asm/pgalloc.h> |
28 | #include <asm/memtype.h> |
29 | #include <asm/setup.h> |
30 | |
31 | #include "physaddr.h" |
32 | |
33 | /* |
34 | * Descriptor controlling ioremap() behavior. |
35 | */ |
36 | struct ioremap_desc { |
37 | unsigned int flags; |
38 | }; |
39 | |
40 | /* |
41 | * Fix up the linear direct mapping of the kernel to avoid cache attribute |
42 | * conflicts. |
43 | */ |
44 | int ioremap_change_attr(unsigned long vaddr, unsigned long size, |
45 | enum page_cache_mode pcm) |
46 | { |
47 | unsigned long nrpages = size >> PAGE_SHIFT; |
48 | int err; |
49 | |
50 | switch (pcm) { |
51 | case _PAGE_CACHE_MODE_UC: |
52 | default: |
53 | err = _set_memory_uc(addr: vaddr, numpages: nrpages); |
54 | break; |
55 | case _PAGE_CACHE_MODE_WC: |
56 | err = _set_memory_wc(addr: vaddr, numpages: nrpages); |
57 | break; |
58 | case _PAGE_CACHE_MODE_WT: |
59 | err = _set_memory_wt(addr: vaddr, numpages: nrpages); |
60 | break; |
61 | case _PAGE_CACHE_MODE_WB: |
62 | err = _set_memory_wb(addr: vaddr, numpages: nrpages); |
63 | break; |
64 | } |
65 | |
66 | return err; |
67 | } |
68 | |
69 | /* Does the range (or a subset of) contain normal RAM? */ |
70 | static unsigned int __ioremap_check_ram(struct resource *res) |
71 | { |
72 | unsigned long start_pfn, stop_pfn; |
73 | unsigned long i; |
74 | |
75 | if ((res->flags & IORESOURCE_SYSTEM_RAM) != IORESOURCE_SYSTEM_RAM) |
76 | return 0; |
77 | |
78 | start_pfn = (res->start + PAGE_SIZE - 1) >> PAGE_SHIFT; |
79 | stop_pfn = (res->end + 1) >> PAGE_SHIFT; |
80 | if (stop_pfn > start_pfn) { |
81 | for (i = 0; i < (stop_pfn - start_pfn); ++i) |
82 | if (pfn_valid(pfn: start_pfn + i) && |
83 | !PageReserved(pfn_to_page(start_pfn + i))) |
84 | return IORES_MAP_SYSTEM_RAM; |
85 | } |
86 | |
87 | return 0; |
88 | } |
89 | |
90 | /* |
91 | * In a SEV guest, NONE and RESERVED should not be mapped encrypted because |
92 | * there the whole memory is already encrypted. |
93 | */ |
94 | static unsigned int __ioremap_check_encrypted(struct resource *res) |
95 | { |
96 | if (!cc_platform_has(attr: CC_ATTR_GUEST_MEM_ENCRYPT)) |
97 | return 0; |
98 | |
99 | switch (res->desc) { |
100 | case IORES_DESC_NONE: |
101 | case IORES_DESC_RESERVED: |
102 | break; |
103 | default: |
104 | return IORES_MAP_ENCRYPTED; |
105 | } |
106 | |
107 | return 0; |
108 | } |
109 | |
110 | /* |
111 | * The EFI runtime services data area is not covered by walk_mem_res(), but must |
112 | * be mapped encrypted when SEV is active. |
113 | */ |
114 | static void __ioremap_check_other(resource_size_t addr, struct ioremap_desc *desc) |
115 | { |
116 | if (!cc_platform_has(attr: CC_ATTR_GUEST_MEM_ENCRYPT)) |
117 | return; |
118 | |
119 | if (x86_platform.hyper.is_private_mmio(addr)) { |
120 | desc->flags |= IORES_MAP_ENCRYPTED; |
121 | return; |
122 | } |
123 | |
124 | if (!IS_ENABLED(CONFIG_EFI)) |
125 | return; |
126 | |
127 | if (efi_mem_type(phys_addr: addr) == EFI_RUNTIME_SERVICES_DATA || |
128 | (efi_mem_type(phys_addr: addr) == EFI_BOOT_SERVICES_DATA && |
129 | efi_mem_attributes(phys_addr: addr) & EFI_MEMORY_RUNTIME)) |
130 | desc->flags |= IORES_MAP_ENCRYPTED; |
131 | } |
132 | |
133 | static int __ioremap_collect_map_flags(struct resource *res, void *arg) |
134 | { |
135 | struct ioremap_desc *desc = arg; |
136 | |
137 | if (!(desc->flags & IORES_MAP_SYSTEM_RAM)) |
138 | desc->flags |= __ioremap_check_ram(res); |
139 | |
140 | if (!(desc->flags & IORES_MAP_ENCRYPTED)) |
141 | desc->flags |= __ioremap_check_encrypted(res); |
142 | |
143 | return ((desc->flags & (IORES_MAP_SYSTEM_RAM | IORES_MAP_ENCRYPTED)) == |
144 | (IORES_MAP_SYSTEM_RAM | IORES_MAP_ENCRYPTED)); |
145 | } |
146 | |
147 | /* |
148 | * To avoid multiple resource walks, this function walks resources marked as |
149 | * IORESOURCE_MEM and IORESOURCE_BUSY and looking for system RAM and/or a |
150 | * resource described not as IORES_DESC_NONE (e.g. IORES_DESC_ACPI_TABLES). |
151 | * |
152 | * After that, deal with misc other ranges in __ioremap_check_other() which do |
153 | * not fall into the above category. |
154 | */ |
155 | static void __ioremap_check_mem(resource_size_t addr, unsigned long size, |
156 | struct ioremap_desc *desc) |
157 | { |
158 | u64 start, end; |
159 | |
160 | start = (u64)addr; |
161 | end = start + size - 1; |
162 | memset(desc, 0, sizeof(struct ioremap_desc)); |
163 | |
164 | walk_mem_res(start, end, arg: desc, func: __ioremap_collect_map_flags); |
165 | |
166 | __ioremap_check_other(addr, desc); |
167 | } |
168 | |
169 | /* |
170 | * Remap an arbitrary physical address space into the kernel virtual |
171 | * address space. It transparently creates kernel huge I/O mapping when |
172 | * the physical address is aligned by a huge page size (1GB or 2MB) and |
173 | * the requested size is at least the huge page size. |
174 | * |
175 | * NOTE: MTRRs can override PAT memory types with a 4KB granularity. |
176 | * Therefore, the mapping code falls back to use a smaller page toward 4KB |
177 | * when a mapping range is covered by non-WB type of MTRRs. |
178 | * |
179 | * NOTE! We need to allow non-page-aligned mappings too: we will obviously |
180 | * have to convert them into an offset in a page-aligned mapping, but the |
181 | * caller shouldn't need to know that small detail. |
182 | */ |
183 | static void __iomem * |
184 | __ioremap_caller(resource_size_t phys_addr, unsigned long size, |
185 | enum page_cache_mode pcm, void *caller, bool encrypted) |
186 | { |
187 | unsigned long offset, vaddr; |
188 | resource_size_t last_addr; |
189 | const resource_size_t unaligned_phys_addr = phys_addr; |
190 | const unsigned long unaligned_size = size; |
191 | struct ioremap_desc io_desc; |
192 | struct vm_struct *area; |
193 | enum page_cache_mode new_pcm; |
194 | pgprot_t prot; |
195 | int retval; |
196 | void __iomem *ret_addr; |
197 | |
198 | /* Don't allow wraparound or zero size */ |
199 | last_addr = phys_addr + size - 1; |
200 | if (!size || last_addr < phys_addr) |
201 | return NULL; |
202 | |
203 | if (!phys_addr_valid(addr: phys_addr)) { |
204 | printk(KERN_WARNING "ioremap: invalid physical address %llx\n" , |
205 | (unsigned long long)phys_addr); |
206 | WARN_ON_ONCE(1); |
207 | return NULL; |
208 | } |
209 | |
210 | __ioremap_check_mem(addr: phys_addr, size, desc: &io_desc); |
211 | |
212 | /* |
213 | * Don't allow anybody to remap normal RAM that we're using.. |
214 | */ |
215 | if (io_desc.flags & IORES_MAP_SYSTEM_RAM) { |
216 | WARN_ONCE(1, "ioremap on RAM at %pa - %pa\n" , |
217 | &phys_addr, &last_addr); |
218 | return NULL; |
219 | } |
220 | |
221 | /* |
222 | * Mappings have to be page-aligned |
223 | */ |
224 | offset = phys_addr & ~PAGE_MASK; |
225 | phys_addr &= PAGE_MASK; |
226 | size = PAGE_ALIGN(last_addr+1) - phys_addr; |
227 | |
228 | /* |
229 | * Mask out any bits not part of the actual physical |
230 | * address, like memory encryption bits. |
231 | */ |
232 | phys_addr &= PHYSICAL_PAGE_MASK; |
233 | |
234 | retval = memtype_reserve(start: phys_addr, end: (u64)phys_addr + size, |
235 | req_pcm: pcm, ret_pcm: &new_pcm); |
236 | if (retval) { |
237 | printk(KERN_ERR "ioremap memtype_reserve failed %d\n" , retval); |
238 | return NULL; |
239 | } |
240 | |
241 | if (pcm != new_pcm) { |
242 | if (!is_new_memtype_allowed(paddr: phys_addr, size, pcm, new_pcm)) { |
243 | printk(KERN_ERR |
244 | "ioremap error for 0x%llx-0x%llx, requested 0x%x, got 0x%x\n" , |
245 | (unsigned long long)phys_addr, |
246 | (unsigned long long)(phys_addr + size), |
247 | pcm, new_pcm); |
248 | goto err_free_memtype; |
249 | } |
250 | pcm = new_pcm; |
251 | } |
252 | |
253 | /* |
254 | * If the page being mapped is in memory and SEV is active then |
255 | * make sure the memory encryption attribute is enabled in the |
256 | * resulting mapping. |
257 | * In TDX guests, memory is marked private by default. If encryption |
258 | * is not requested (using encrypted), explicitly set decrypt |
259 | * attribute in all IOREMAPPED memory. |
260 | */ |
261 | prot = PAGE_KERNEL_IO; |
262 | if ((io_desc.flags & IORES_MAP_ENCRYPTED) || encrypted) |
263 | prot = pgprot_encrypted(prot); |
264 | else |
265 | prot = pgprot_decrypted(prot); |
266 | |
267 | switch (pcm) { |
268 | case _PAGE_CACHE_MODE_UC: |
269 | default: |
270 | prot = __pgprot(pgprot_val(prot) | |
271 | cachemode2protval(_PAGE_CACHE_MODE_UC)); |
272 | break; |
273 | case _PAGE_CACHE_MODE_UC_MINUS: |
274 | prot = __pgprot(pgprot_val(prot) | |
275 | cachemode2protval(_PAGE_CACHE_MODE_UC_MINUS)); |
276 | break; |
277 | case _PAGE_CACHE_MODE_WC: |
278 | prot = __pgprot(pgprot_val(prot) | |
279 | cachemode2protval(_PAGE_CACHE_MODE_WC)); |
280 | break; |
281 | case _PAGE_CACHE_MODE_WT: |
282 | prot = __pgprot(pgprot_val(prot) | |
283 | cachemode2protval(_PAGE_CACHE_MODE_WT)); |
284 | break; |
285 | case _PAGE_CACHE_MODE_WB: |
286 | break; |
287 | } |
288 | |
289 | /* |
290 | * Ok, go for it.. |
291 | */ |
292 | area = get_vm_area_caller(size, VM_IOREMAP, caller); |
293 | if (!area) |
294 | goto err_free_memtype; |
295 | area->phys_addr = phys_addr; |
296 | vaddr = (unsigned long) area->addr; |
297 | |
298 | if (memtype_kernel_map_sync(base: phys_addr, size, pcm)) |
299 | goto err_free_area; |
300 | |
301 | if (ioremap_page_range(addr: vaddr, end: vaddr + size, phys_addr, prot)) |
302 | goto err_free_area; |
303 | |
304 | ret_addr = (void __iomem *) (vaddr + offset); |
305 | mmiotrace_ioremap(offset: unaligned_phys_addr, size: unaligned_size, addr: ret_addr); |
306 | |
307 | /* |
308 | * Check if the request spans more than any BAR in the iomem resource |
309 | * tree. |
310 | */ |
311 | if (iomem_map_sanity_check(addr: unaligned_phys_addr, size: unaligned_size)) |
312 | pr_warn("caller %pS mapping multiple BARs\n" , caller); |
313 | |
314 | return ret_addr; |
315 | err_free_area: |
316 | free_vm_area(area); |
317 | err_free_memtype: |
318 | memtype_free(start: phys_addr, end: phys_addr + size); |
319 | return NULL; |
320 | } |
321 | |
322 | /** |
323 | * ioremap - map bus memory into CPU space |
324 | * @phys_addr: bus address of the memory |
325 | * @size: size of the resource to map |
326 | * |
327 | * ioremap performs a platform specific sequence of operations to |
328 | * make bus memory CPU accessible via the readb/readw/readl/writeb/ |
329 | * writew/writel functions and the other mmio helpers. The returned |
330 | * address is not guaranteed to be usable directly as a virtual |
331 | * address. |
332 | * |
333 | * This version of ioremap ensures that the memory is marked uncachable |
334 | * on the CPU as well as honouring existing caching rules from things like |
335 | * the PCI bus. Note that there are other caches and buffers on many |
336 | * busses. In particular driver authors should read up on PCI writes |
337 | * |
338 | * It's useful if some control registers are in such an area and |
339 | * write combining or read caching is not desirable: |
340 | * |
341 | * Must be freed with iounmap. |
342 | */ |
343 | void __iomem *ioremap(resource_size_t phys_addr, unsigned long size) |
344 | { |
345 | /* |
346 | * Ideally, this should be: |
347 | * pat_enabled() ? _PAGE_CACHE_MODE_UC : _PAGE_CACHE_MODE_UC_MINUS; |
348 | * |
349 | * Till we fix all X drivers to use ioremap_wc(), we will use |
350 | * UC MINUS. Drivers that are certain they need or can already |
351 | * be converted over to strong UC can use ioremap_uc(). |
352 | */ |
353 | enum page_cache_mode pcm = _PAGE_CACHE_MODE_UC_MINUS; |
354 | |
355 | return __ioremap_caller(phys_addr, size, pcm, |
356 | caller: __builtin_return_address(0), encrypted: false); |
357 | } |
358 | EXPORT_SYMBOL(ioremap); |
359 | |
360 | /** |
361 | * ioremap_uc - map bus memory into CPU space as strongly uncachable |
362 | * @phys_addr: bus address of the memory |
363 | * @size: size of the resource to map |
364 | * |
365 | * ioremap_uc performs a platform specific sequence of operations to |
366 | * make bus memory CPU accessible via the readb/readw/readl/writeb/ |
367 | * writew/writel functions and the other mmio helpers. The returned |
368 | * address is not guaranteed to be usable directly as a virtual |
369 | * address. |
370 | * |
371 | * This version of ioremap ensures that the memory is marked with a strong |
372 | * preference as completely uncachable on the CPU when possible. For non-PAT |
373 | * systems this ends up setting page-attribute flags PCD=1, PWT=1. For PAT |
374 | * systems this will set the PAT entry for the pages as strong UC. This call |
375 | * will honor existing caching rules from things like the PCI bus. Note that |
376 | * there are other caches and buffers on many busses. In particular driver |
377 | * authors should read up on PCI writes. |
378 | * |
379 | * It's useful if some control registers are in such an area and |
380 | * write combining or read caching is not desirable: |
381 | * |
382 | * Must be freed with iounmap. |
383 | */ |
384 | void __iomem *ioremap_uc(resource_size_t phys_addr, unsigned long size) |
385 | { |
386 | enum page_cache_mode pcm = _PAGE_CACHE_MODE_UC; |
387 | |
388 | return __ioremap_caller(phys_addr, size, pcm, |
389 | caller: __builtin_return_address(0), encrypted: false); |
390 | } |
391 | EXPORT_SYMBOL_GPL(ioremap_uc); |
392 | |
393 | /** |
394 | * ioremap_wc - map memory into CPU space write combined |
395 | * @phys_addr: bus address of the memory |
396 | * @size: size of the resource to map |
397 | * |
398 | * This version of ioremap ensures that the memory is marked write combining. |
399 | * Write combining allows faster writes to some hardware devices. |
400 | * |
401 | * Must be freed with iounmap. |
402 | */ |
403 | void __iomem *ioremap_wc(resource_size_t phys_addr, unsigned long size) |
404 | { |
405 | return __ioremap_caller(phys_addr, size, pcm: _PAGE_CACHE_MODE_WC, |
406 | caller: __builtin_return_address(0), encrypted: false); |
407 | } |
408 | EXPORT_SYMBOL(ioremap_wc); |
409 | |
410 | /** |
411 | * ioremap_wt - map memory into CPU space write through |
412 | * @phys_addr: bus address of the memory |
413 | * @size: size of the resource to map |
414 | * |
415 | * This version of ioremap ensures that the memory is marked write through. |
416 | * Write through stores data into memory while keeping the cache up-to-date. |
417 | * |
418 | * Must be freed with iounmap. |
419 | */ |
420 | void __iomem *ioremap_wt(resource_size_t phys_addr, unsigned long size) |
421 | { |
422 | return __ioremap_caller(phys_addr, size, pcm: _PAGE_CACHE_MODE_WT, |
423 | caller: __builtin_return_address(0), encrypted: false); |
424 | } |
425 | EXPORT_SYMBOL(ioremap_wt); |
426 | |
427 | void __iomem *ioremap_encrypted(resource_size_t phys_addr, unsigned long size) |
428 | { |
429 | return __ioremap_caller(phys_addr, size, pcm: _PAGE_CACHE_MODE_WB, |
430 | caller: __builtin_return_address(0), encrypted: true); |
431 | } |
432 | EXPORT_SYMBOL(ioremap_encrypted); |
433 | |
434 | void __iomem *ioremap_cache(resource_size_t phys_addr, unsigned long size) |
435 | { |
436 | return __ioremap_caller(phys_addr, size, pcm: _PAGE_CACHE_MODE_WB, |
437 | caller: __builtin_return_address(0), encrypted: false); |
438 | } |
439 | EXPORT_SYMBOL(ioremap_cache); |
440 | |
441 | void __iomem *ioremap_prot(resource_size_t phys_addr, unsigned long size, |
442 | unsigned long prot_val) |
443 | { |
444 | return __ioremap_caller(phys_addr, size, |
445 | pcm: pgprot2cachemode(__pgprot(prot_val)), |
446 | caller: __builtin_return_address(0), encrypted: false); |
447 | } |
448 | EXPORT_SYMBOL(ioremap_prot); |
449 | |
450 | /** |
451 | * iounmap - Free a IO remapping |
452 | * @addr: virtual address from ioremap_* |
453 | * |
454 | * Caller must ensure there is only one unmapping for the same pointer. |
455 | */ |
456 | void iounmap(volatile void __iomem *addr) |
457 | { |
458 | struct vm_struct *p, *o; |
459 | |
460 | if ((void __force *)addr <= high_memory) |
461 | return; |
462 | |
463 | /* |
464 | * The PCI/ISA range special-casing was removed from __ioremap() |
465 | * so this check, in theory, can be removed. However, there are |
466 | * cases where iounmap() is called for addresses not obtained via |
467 | * ioremap() (vga16fb for example). Add a warning so that these |
468 | * cases can be caught and fixed. |
469 | */ |
470 | if ((void __force *)addr >= phys_to_virt(ISA_START_ADDRESS) && |
471 | (void __force *)addr < phys_to_virt(ISA_END_ADDRESS)) { |
472 | WARN(1, "iounmap() called for ISA range not obtained using ioremap()\n" ); |
473 | return; |
474 | } |
475 | |
476 | mmiotrace_iounmap(addr); |
477 | |
478 | addr = (volatile void __iomem *) |
479 | (PAGE_MASK & (unsigned long __force)addr); |
480 | |
481 | /* Use the vm area unlocked, assuming the caller |
482 | ensures there isn't another iounmap for the same address |
483 | in parallel. Reuse of the virtual address is prevented by |
484 | leaving it in the global lists until we're done with it. |
485 | cpa takes care of the direct mappings. */ |
486 | p = find_vm_area(addr: (void __force *)addr); |
487 | |
488 | if (!p) { |
489 | printk(KERN_ERR "iounmap: bad address %p\n" , addr); |
490 | dump_stack(); |
491 | return; |
492 | } |
493 | |
494 | kmsan_iounmap_page_range(start: (unsigned long)addr, |
495 | end: (unsigned long)addr + get_vm_area_size(area: p)); |
496 | memtype_free(start: p->phys_addr, end: p->phys_addr + get_vm_area_size(area: p)); |
497 | |
498 | /* Finally remove it */ |
499 | o = remove_vm_area(addr: (void __force *)addr); |
500 | BUG_ON(p != o || o == NULL); |
501 | kfree(objp: p); |
502 | } |
503 | EXPORT_SYMBOL(iounmap); |
504 | |
505 | /* |
506 | * Convert a physical pointer to a virtual kernel pointer for /dev/mem |
507 | * access |
508 | */ |
509 | void *xlate_dev_mem_ptr(phys_addr_t phys) |
510 | { |
511 | unsigned long start = phys & PAGE_MASK; |
512 | unsigned long offset = phys & ~PAGE_MASK; |
513 | void *vaddr; |
514 | |
515 | /* memremap() maps if RAM, otherwise falls back to ioremap() */ |
516 | vaddr = memremap(offset: start, PAGE_SIZE, flags: MEMREMAP_WB); |
517 | |
518 | /* Only add the offset on success and return NULL if memremap() failed */ |
519 | if (vaddr) |
520 | vaddr += offset; |
521 | |
522 | return vaddr; |
523 | } |
524 | |
525 | void unxlate_dev_mem_ptr(phys_addr_t phys, void *addr) |
526 | { |
527 | memunmap(addr: (void *)((unsigned long)addr & PAGE_MASK)); |
528 | } |
529 | |
530 | #ifdef CONFIG_AMD_MEM_ENCRYPT |
531 | /* |
532 | * Examine the physical address to determine if it is an area of memory |
533 | * that should be mapped decrypted. If the memory is not part of the |
534 | * kernel usable area it was accessed and created decrypted, so these |
535 | * areas should be mapped decrypted. And since the encryption key can |
536 | * change across reboots, persistent memory should also be mapped |
537 | * decrypted. |
538 | * |
539 | * If SEV is active, that implies that BIOS/UEFI also ran encrypted so |
540 | * only persistent memory should be mapped decrypted. |
541 | */ |
542 | static bool memremap_should_map_decrypted(resource_size_t phys_addr, |
543 | unsigned long size) |
544 | { |
545 | int is_pmem; |
546 | |
547 | /* |
548 | * Check if the address is part of a persistent memory region. |
549 | * This check covers areas added by E820, EFI and ACPI. |
550 | */ |
551 | is_pmem = region_intersects(offset: phys_addr, size, IORESOURCE_MEM, |
552 | desc: IORES_DESC_PERSISTENT_MEMORY); |
553 | if (is_pmem != REGION_DISJOINT) |
554 | return true; |
555 | |
556 | /* |
557 | * Check if the non-volatile attribute is set for an EFI |
558 | * reserved area. |
559 | */ |
560 | if (efi_enabled(EFI_BOOT)) { |
561 | switch (efi_mem_type(phys_addr)) { |
562 | case EFI_RESERVED_TYPE: |
563 | if (efi_mem_attributes(phys_addr) & EFI_MEMORY_NV) |
564 | return true; |
565 | break; |
566 | default: |
567 | break; |
568 | } |
569 | } |
570 | |
571 | /* Check if the address is outside kernel usable area */ |
572 | switch (e820__get_entry_type(start: phys_addr, end: phys_addr + size - 1)) { |
573 | case E820_TYPE_RESERVED: |
574 | case E820_TYPE_ACPI: |
575 | case E820_TYPE_NVS: |
576 | case E820_TYPE_UNUSABLE: |
577 | /* For SEV, these areas are encrypted */ |
578 | if (cc_platform_has(attr: CC_ATTR_GUEST_MEM_ENCRYPT)) |
579 | break; |
580 | fallthrough; |
581 | |
582 | case E820_TYPE_PRAM: |
583 | return true; |
584 | default: |
585 | break; |
586 | } |
587 | |
588 | return false; |
589 | } |
590 | |
591 | /* |
592 | * Examine the physical address to determine if it is EFI data. Check |
593 | * it against the boot params structure and EFI tables and memory types. |
594 | */ |
595 | static bool memremap_is_efi_data(resource_size_t phys_addr, |
596 | unsigned long size) |
597 | { |
598 | u64 paddr; |
599 | |
600 | /* Check if the address is part of EFI boot/runtime data */ |
601 | if (!efi_enabled(EFI_BOOT)) |
602 | return false; |
603 | |
604 | paddr = boot_params.efi_info.efi_memmap_hi; |
605 | paddr <<= 32; |
606 | paddr |= boot_params.efi_info.efi_memmap; |
607 | if (phys_addr == paddr) |
608 | return true; |
609 | |
610 | paddr = boot_params.efi_info.efi_systab_hi; |
611 | paddr <<= 32; |
612 | paddr |= boot_params.efi_info.efi_systab; |
613 | if (phys_addr == paddr) |
614 | return true; |
615 | |
616 | if (efi_is_table_address(phys_addr)) |
617 | return true; |
618 | |
619 | switch (efi_mem_type(phys_addr)) { |
620 | case EFI_BOOT_SERVICES_DATA: |
621 | case EFI_RUNTIME_SERVICES_DATA: |
622 | return true; |
623 | default: |
624 | break; |
625 | } |
626 | |
627 | return false; |
628 | } |
629 | |
630 | /* |
631 | * Examine the physical address to determine if it is boot data by checking |
632 | * it against the boot params setup_data chain. |
633 | */ |
634 | static bool memremap_is_setup_data(resource_size_t phys_addr, |
635 | unsigned long size) |
636 | { |
637 | struct setup_indirect *indirect; |
638 | struct setup_data *data; |
639 | u64 paddr, paddr_next; |
640 | |
641 | paddr = boot_params.hdr.setup_data; |
642 | while (paddr) { |
643 | unsigned int len; |
644 | |
645 | if (phys_addr == paddr) |
646 | return true; |
647 | |
648 | data = memremap(offset: paddr, size: sizeof(*data), |
649 | flags: MEMREMAP_WB | MEMREMAP_DEC); |
650 | if (!data) { |
651 | pr_warn("failed to memremap setup_data entry\n" ); |
652 | return false; |
653 | } |
654 | |
655 | paddr_next = data->next; |
656 | len = data->len; |
657 | |
658 | if ((phys_addr > paddr) && (phys_addr < (paddr + len))) { |
659 | memunmap(addr: data); |
660 | return true; |
661 | } |
662 | |
663 | if (data->type == SETUP_INDIRECT) { |
664 | memunmap(addr: data); |
665 | data = memremap(offset: paddr, size: sizeof(*data) + len, |
666 | flags: MEMREMAP_WB | MEMREMAP_DEC); |
667 | if (!data) { |
668 | pr_warn("failed to memremap indirect setup_data\n" ); |
669 | return false; |
670 | } |
671 | |
672 | indirect = (struct setup_indirect *)data->data; |
673 | |
674 | if (indirect->type != SETUP_INDIRECT) { |
675 | paddr = indirect->addr; |
676 | len = indirect->len; |
677 | } |
678 | } |
679 | |
680 | memunmap(addr: data); |
681 | |
682 | if ((phys_addr > paddr) && (phys_addr < (paddr + len))) |
683 | return true; |
684 | |
685 | paddr = paddr_next; |
686 | } |
687 | |
688 | return false; |
689 | } |
690 | |
691 | /* |
692 | * Examine the physical address to determine if it is boot data by checking |
693 | * it against the boot params setup_data chain (early boot version). |
694 | */ |
695 | static bool __init early_memremap_is_setup_data(resource_size_t phys_addr, |
696 | unsigned long size) |
697 | { |
698 | struct setup_indirect *indirect; |
699 | struct setup_data *data; |
700 | u64 paddr, paddr_next; |
701 | |
702 | paddr = boot_params.hdr.setup_data; |
703 | while (paddr) { |
704 | unsigned int len, size; |
705 | |
706 | if (phys_addr == paddr) |
707 | return true; |
708 | |
709 | data = early_memremap_decrypted(phys_addr: paddr, size: sizeof(*data)); |
710 | if (!data) { |
711 | pr_warn("failed to early memremap setup_data entry\n" ); |
712 | return false; |
713 | } |
714 | |
715 | size = sizeof(*data); |
716 | |
717 | paddr_next = data->next; |
718 | len = data->len; |
719 | |
720 | if ((phys_addr > paddr) && (phys_addr < (paddr + len))) { |
721 | early_memunmap(addr: data, size: sizeof(*data)); |
722 | return true; |
723 | } |
724 | |
725 | if (data->type == SETUP_INDIRECT) { |
726 | size += len; |
727 | early_memunmap(addr: data, size: sizeof(*data)); |
728 | data = early_memremap_decrypted(phys_addr: paddr, size); |
729 | if (!data) { |
730 | pr_warn("failed to early memremap indirect setup_data\n" ); |
731 | return false; |
732 | } |
733 | |
734 | indirect = (struct setup_indirect *)data->data; |
735 | |
736 | if (indirect->type != SETUP_INDIRECT) { |
737 | paddr = indirect->addr; |
738 | len = indirect->len; |
739 | } |
740 | } |
741 | |
742 | early_memunmap(addr: data, size); |
743 | |
744 | if ((phys_addr > paddr) && (phys_addr < (paddr + len))) |
745 | return true; |
746 | |
747 | paddr = paddr_next; |
748 | } |
749 | |
750 | return false; |
751 | } |
752 | |
753 | /* |
754 | * Architecture function to determine if RAM remap is allowed. By default, a |
755 | * RAM remap will map the data as encrypted. Determine if a RAM remap should |
756 | * not be done so that the data will be mapped decrypted. |
757 | */ |
758 | bool arch_memremap_can_ram_remap(resource_size_t phys_addr, unsigned long size, |
759 | unsigned long flags) |
760 | { |
761 | if (!cc_platform_has(attr: CC_ATTR_MEM_ENCRYPT)) |
762 | return true; |
763 | |
764 | if (flags & MEMREMAP_ENC) |
765 | return true; |
766 | |
767 | if (flags & MEMREMAP_DEC) |
768 | return false; |
769 | |
770 | if (cc_platform_has(attr: CC_ATTR_HOST_MEM_ENCRYPT)) { |
771 | if (memremap_is_setup_data(phys_addr, size) || |
772 | memremap_is_efi_data(phys_addr, size)) |
773 | return false; |
774 | } |
775 | |
776 | return !memremap_should_map_decrypted(phys_addr, size); |
777 | } |
778 | |
779 | /* |
780 | * Architecture override of __weak function to adjust the protection attributes |
781 | * used when remapping memory. By default, early_memremap() will map the data |
782 | * as encrypted. Determine if an encrypted mapping should not be done and set |
783 | * the appropriate protection attributes. |
784 | */ |
785 | pgprot_t __init early_memremap_pgprot_adjust(resource_size_t phys_addr, |
786 | unsigned long size, |
787 | pgprot_t prot) |
788 | { |
789 | bool encrypted_prot; |
790 | |
791 | if (!cc_platform_has(attr: CC_ATTR_MEM_ENCRYPT)) |
792 | return prot; |
793 | |
794 | encrypted_prot = true; |
795 | |
796 | if (cc_platform_has(attr: CC_ATTR_HOST_MEM_ENCRYPT)) { |
797 | if (early_memremap_is_setup_data(phys_addr, size) || |
798 | memremap_is_efi_data(phys_addr, size)) |
799 | encrypted_prot = false; |
800 | } |
801 | |
802 | if (encrypted_prot && memremap_should_map_decrypted(phys_addr, size)) |
803 | encrypted_prot = false; |
804 | |
805 | return encrypted_prot ? pgprot_encrypted(prot) |
806 | : pgprot_decrypted(prot); |
807 | } |
808 | |
809 | bool phys_mem_access_encrypted(unsigned long phys_addr, unsigned long size) |
810 | { |
811 | return arch_memremap_can_ram_remap(phys_addr, size, flags: 0); |
812 | } |
813 | |
814 | /* Remap memory with encryption */ |
815 | void __init *early_memremap_encrypted(resource_size_t phys_addr, |
816 | unsigned long size) |
817 | { |
818 | return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_ENC); |
819 | } |
820 | |
821 | /* |
822 | * Remap memory with encryption and write-protected - cannot be called |
823 | * before pat_init() is called |
824 | */ |
825 | void __init *early_memremap_encrypted_wp(resource_size_t phys_addr, |
826 | unsigned long size) |
827 | { |
828 | if (!x86_has_pat_wp()) |
829 | return NULL; |
830 | return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_ENC_WP); |
831 | } |
832 | |
833 | /* Remap memory without encryption */ |
834 | void __init *early_memremap_decrypted(resource_size_t phys_addr, |
835 | unsigned long size) |
836 | { |
837 | return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_NOENC); |
838 | } |
839 | |
840 | /* |
841 | * Remap memory without encryption and write-protected - cannot be called |
842 | * before pat_init() is called |
843 | */ |
844 | void __init *early_memremap_decrypted_wp(resource_size_t phys_addr, |
845 | unsigned long size) |
846 | { |
847 | if (!x86_has_pat_wp()) |
848 | return NULL; |
849 | return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_NOENC_WP); |
850 | } |
851 | #endif /* CONFIG_AMD_MEM_ENCRYPT */ |
852 | |
853 | static pte_t bm_pte[PAGE_SIZE/sizeof(pte_t)] __page_aligned_bss; |
854 | |
855 | static inline pmd_t * __init early_ioremap_pmd(unsigned long addr) |
856 | { |
857 | /* Don't assume we're using swapper_pg_dir at this point */ |
858 | pgd_t *base = __va(read_cr3_pa()); |
859 | pgd_t *pgd = &base[pgd_index(addr)]; |
860 | p4d_t *p4d = p4d_offset(pgd, address: addr); |
861 | pud_t *pud = pud_offset(p4d, address: addr); |
862 | pmd_t *pmd = pmd_offset(pud, address: addr); |
863 | |
864 | return pmd; |
865 | } |
866 | |
867 | static inline pte_t * __init early_ioremap_pte(unsigned long addr) |
868 | { |
869 | return &bm_pte[pte_index(address: addr)]; |
870 | } |
871 | |
872 | bool __init is_early_ioremap_ptep(pte_t *ptep) |
873 | { |
874 | return ptep >= &bm_pte[0] && ptep < &bm_pte[PAGE_SIZE/sizeof(pte_t)]; |
875 | } |
876 | |
877 | void __init early_ioremap_init(void) |
878 | { |
879 | pmd_t *pmd; |
880 | |
881 | #ifdef CONFIG_X86_64 |
882 | BUILD_BUG_ON((fix_to_virt(0) + PAGE_SIZE) & ((1 << PMD_SHIFT) - 1)); |
883 | #else |
884 | WARN_ON((fix_to_virt(0) + PAGE_SIZE) & ((1 << PMD_SHIFT) - 1)); |
885 | #endif |
886 | |
887 | early_ioremap_setup(); |
888 | |
889 | pmd = early_ioremap_pmd(addr: fix_to_virt(idx: FIX_BTMAP_BEGIN)); |
890 | memset(bm_pte, 0, sizeof(bm_pte)); |
891 | pmd_populate_kernel(mm: &init_mm, pmd, pte: bm_pte); |
892 | |
893 | /* |
894 | * The boot-ioremap range spans multiple pmds, for which |
895 | * we are not prepared: |
896 | */ |
897 | #define __FIXADDR_TOP (-PAGE_SIZE) |
898 | BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT) |
899 | != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT)); |
900 | #undef __FIXADDR_TOP |
901 | if (pmd != early_ioremap_pmd(addr: fix_to_virt(idx: FIX_BTMAP_END))) { |
902 | WARN_ON(1); |
903 | printk(KERN_WARNING "pmd %p != %p\n" , |
904 | pmd, early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END))); |
905 | printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n" , |
906 | fix_to_virt(FIX_BTMAP_BEGIN)); |
907 | printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END): %08lx\n" , |
908 | fix_to_virt(FIX_BTMAP_END)); |
909 | |
910 | printk(KERN_WARNING "FIX_BTMAP_END: %d\n" , FIX_BTMAP_END); |
911 | printk(KERN_WARNING "FIX_BTMAP_BEGIN: %d\n" , |
912 | FIX_BTMAP_BEGIN); |
913 | } |
914 | } |
915 | |
916 | void __init __early_set_fixmap(enum fixed_addresses idx, |
917 | phys_addr_t phys, pgprot_t flags) |
918 | { |
919 | unsigned long addr = __fix_to_virt(idx); |
920 | pte_t *pte; |
921 | |
922 | if (idx >= __end_of_fixed_addresses) { |
923 | BUG(); |
924 | return; |
925 | } |
926 | pte = early_ioremap_pte(addr); |
927 | |
928 | /* Sanitize 'prot' against any unsupported bits: */ |
929 | pgprot_val(flags) &= __supported_pte_mask; |
930 | |
931 | if (pgprot_val(flags)) |
932 | set_pte(ptep: pte, pte: pfn_pte(page_nr: phys >> PAGE_SHIFT, pgprot: flags)); |
933 | else |
934 | pte_clear(mm: &init_mm, addr, ptep: pte); |
935 | flush_tlb_one_kernel(addr); |
936 | } |
937 | |