1// SPDX-License-Identifier: GPL-2.0-only
2
3#include <linux/efi.h>
4#include <linux/memblock.h>
5#include <linux/spinlock.h>
6#include <linux/crash_dump.h>
7#include <asm/unaccepted_memory.h>
8
9/* Protects unaccepted memory bitmap and accepting_list */
10static DEFINE_SPINLOCK(unaccepted_memory_lock);
11
12struct accept_range {
13 struct list_head list;
14 unsigned long start;
15 unsigned long end;
16};
17
18static LIST_HEAD(accepting_list);
19
20/*
21 * accept_memory() -- Consult bitmap and accept the memory if needed.
22 *
23 * Only memory that is explicitly marked as unaccepted in the bitmap requires
24 * an action. All the remaining memory is implicitly accepted and doesn't need
25 * acceptance.
26 *
27 * No need to accept:
28 * - anything if the system has no unaccepted table;
29 * - memory that is below phys_base;
30 * - memory that is above the memory that addressable by the bitmap;
31 */
32void accept_memory(phys_addr_t start, phys_addr_t end)
33{
34 struct efi_unaccepted_memory *unaccepted;
35 unsigned long range_start, range_end;
36 struct accept_range range, *entry;
37 unsigned long flags;
38 u64 unit_size;
39
40 unaccepted = efi_get_unaccepted_table();
41 if (!unaccepted)
42 return;
43
44 unit_size = unaccepted->unit_size;
45
46 /*
47 * Only care for the part of the range that is represented
48 * in the bitmap.
49 */
50 if (start < unaccepted->phys_base)
51 start = unaccepted->phys_base;
52 if (end < unaccepted->phys_base)
53 return;
54
55 /* Translate to offsets from the beginning of the bitmap */
56 start -= unaccepted->phys_base;
57 end -= unaccepted->phys_base;
58
59 /*
60 * load_unaligned_zeropad() can lead to unwanted loads across page
61 * boundaries. The unwanted loads are typically harmless. But, they
62 * might be made to totally unrelated or even unmapped memory.
63 * load_unaligned_zeropad() relies on exception fixup (#PF, #GP and now
64 * #VE) to recover from these unwanted loads.
65 *
66 * But, this approach does not work for unaccepted memory. For TDX, a
67 * load from unaccepted memory will not lead to a recoverable exception
68 * within the guest. The guest will exit to the VMM where the only
69 * recourse is to terminate the guest.
70 *
71 * There are two parts to fix this issue and comprehensively avoid
72 * access to unaccepted memory. Together these ensure that an extra
73 * "guard" page is accepted in addition to the memory that needs to be
74 * used:
75 *
76 * 1. Implicitly extend the range_contains_unaccepted_memory(start, end)
77 * checks up to end+unit_size if 'end' is aligned on a unit_size
78 * boundary.
79 *
80 * 2. Implicitly extend accept_memory(start, end) to end+unit_size if
81 * 'end' is aligned on a unit_size boundary. (immediately following
82 * this comment)
83 */
84 if (!(end % unit_size))
85 end += unit_size;
86
87 /* Make sure not to overrun the bitmap */
88 if (end > unaccepted->size * unit_size * BITS_PER_BYTE)
89 end = unaccepted->size * unit_size * BITS_PER_BYTE;
90
91 range.start = start / unit_size;
92 range.end = DIV_ROUND_UP(end, unit_size);
93retry:
94 spin_lock_irqsave(&unaccepted_memory_lock, flags);
95
96 /*
97 * Check if anybody works on accepting the same range of the memory.
98 *
99 * The check is done with unit_size granularity. It is crucial to catch
100 * all accept requests to the same unit_size block, even if they don't
101 * overlap on physical address level.
102 */
103 list_for_each_entry(entry, &accepting_list, list) {
104 if (entry->end <= range.start)
105 continue;
106 if (entry->start >= range.end)
107 continue;
108
109 /*
110 * Somebody else accepting the range. Or at least part of it.
111 *
112 * Drop the lock and retry until it is complete.
113 */
114 spin_unlock_irqrestore(lock: &unaccepted_memory_lock, flags);
115 goto retry;
116 }
117
118 /*
119 * Register that the range is about to be accepted.
120 * Make sure nobody else will accept it.
121 */
122 list_add(new: &range.list, head: &accepting_list);
123
124 range_start = range.start;
125 for_each_set_bitrange_from(range_start, range_end, unaccepted->bitmap,
126 range.end) {
127 unsigned long phys_start, phys_end;
128 unsigned long len = range_end - range_start;
129
130 phys_start = range_start * unit_size + unaccepted->phys_base;
131 phys_end = range_end * unit_size + unaccepted->phys_base;
132
133 /*
134 * Keep interrupts disabled until the accept operation is
135 * complete in order to prevent deadlocks.
136 *
137 * Enabling interrupts before calling arch_accept_memory()
138 * creates an opportunity for an interrupt handler to request
139 * acceptance for the same memory. The handler will continuously
140 * spin with interrupts disabled, preventing other task from
141 * making progress with the acceptance process.
142 */
143 spin_unlock(lock: &unaccepted_memory_lock);
144
145 arch_accept_memory(start: phys_start, end: phys_end);
146
147 spin_lock(lock: &unaccepted_memory_lock);
148 bitmap_clear(map: unaccepted->bitmap, start: range_start, nbits: len);
149 }
150
151 list_del(entry: &range.list);
152 spin_unlock_irqrestore(lock: &unaccepted_memory_lock, flags);
153}
154
155bool range_contains_unaccepted_memory(phys_addr_t start, phys_addr_t end)
156{
157 struct efi_unaccepted_memory *unaccepted;
158 unsigned long flags;
159 bool ret = false;
160 u64 unit_size;
161
162 unaccepted = efi_get_unaccepted_table();
163 if (!unaccepted)
164 return false;
165
166 unit_size = unaccepted->unit_size;
167
168 /*
169 * Only care for the part of the range that is represented
170 * in the bitmap.
171 */
172 if (start < unaccepted->phys_base)
173 start = unaccepted->phys_base;
174 if (end < unaccepted->phys_base)
175 return false;
176
177 /* Translate to offsets from the beginning of the bitmap */
178 start -= unaccepted->phys_base;
179 end -= unaccepted->phys_base;
180
181 /*
182 * Also consider the unaccepted state of the *next* page. See fix #1 in
183 * the comment on load_unaligned_zeropad() in accept_memory().
184 */
185 if (!(end % unit_size))
186 end += unit_size;
187
188 /* Make sure not to overrun the bitmap */
189 if (end > unaccepted->size * unit_size * BITS_PER_BYTE)
190 end = unaccepted->size * unit_size * BITS_PER_BYTE;
191
192 spin_lock_irqsave(&unaccepted_memory_lock, flags);
193 while (start < end) {
194 if (test_bit(start / unit_size, unaccepted->bitmap)) {
195 ret = true;
196 break;
197 }
198
199 start += unit_size;
200 }
201 spin_unlock_irqrestore(lock: &unaccepted_memory_lock, flags);
202
203 return ret;
204}
205
206#ifdef CONFIG_PROC_VMCORE
207static bool unaccepted_memory_vmcore_pfn_is_ram(struct vmcore_cb *cb,
208 unsigned long pfn)
209{
210 return !pfn_is_unaccepted_memory(pfn);
211}
212
213static struct vmcore_cb vmcore_cb = {
214 .pfn_is_ram = unaccepted_memory_vmcore_pfn_is_ram,
215};
216
217static int __init unaccepted_memory_init_kdump(void)
218{
219 register_vmcore_cb(cb: &vmcore_cb);
220 return 0;
221}
222core_initcall(unaccepted_memory_init_kdump);
223#endif /* CONFIG_PROC_VMCORE */
224

source code of linux/drivers/firmware/efi/unaccepted_memory.c