1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
2 | /* |
3 | * relocate_kernel.S - put the kernel image in place to boot |
4 | * Copyright (C) 2002-2005 Eric Biederman <ebiederm@xmission.com> |
5 | */ |
6 | |
7 | #include <linux/linkage.h> |
8 | #include <asm/page_types.h> |
9 | #include <asm/kexec.h> |
10 | #include <asm/processor-flags.h> |
11 | #include <asm/pgtable_types.h> |
12 | #include <asm/nospec-branch.h> |
13 | #include <asm/unwind_hints.h> |
14 | |
15 | /* |
16 | * Must be relocatable PIC code callable as a C function, in particular |
17 | * there must be a plain RET and not jump to return thunk. |
18 | */ |
19 | |
20 | #define PTR(x) (x << 3) |
21 | #define PAGE_ATTR (_PAGE_PRESENT | _PAGE_RW | _PAGE_ACCESSED | _PAGE_DIRTY) |
22 | |
23 | /* |
24 | * control_page + KEXEC_CONTROL_CODE_MAX_SIZE |
25 | * ~ control_page + PAGE_SIZE are used as data storage and stack for |
26 | * jumping back |
27 | */ |
28 | #define DATA(offset) (KEXEC_CONTROL_CODE_MAX_SIZE+(offset)) |
29 | |
30 | /* Minimal CPU state */ |
31 | #define RSP DATA(0x0) |
32 | #define CR0 DATA(0x8) |
33 | #define CR3 DATA(0x10) |
34 | #define CR4 DATA(0x18) |
35 | |
36 | /* other data */ |
37 | #define CP_PA_TABLE_PAGE DATA(0x20) |
38 | #define CP_PA_SWAP_PAGE DATA(0x28) |
39 | #define CP_PA_BACKUP_PAGES_MAP DATA(0x30) |
40 | |
41 | .text |
42 | .align PAGE_SIZE |
43 | .code64 |
44 | SYM_CODE_START_NOALIGN(relocate_range) |
45 | SYM_CODE_START_NOALIGN(relocate_kernel) |
46 | UNWIND_HINT_END_OF_STACK |
47 | ANNOTATE_NOENDBR |
48 | /* |
49 | * %rdi indirection_page |
50 | * %rsi page_list |
51 | * %rdx start address |
52 | * %rcx preserve_context |
53 | * %r8 host_mem_enc_active |
54 | */ |
55 | |
56 | /* Save the CPU context, used for jumping back */ |
57 | pushq %rbx |
58 | pushq %rbp |
59 | pushq %r12 |
60 | pushq %r13 |
61 | pushq %r14 |
62 | pushq %r15 |
63 | pushf |
64 | |
65 | movq PTR(VA_CONTROL_PAGE)(%rsi), %r11 |
66 | movq %rsp, RSP(%r11) |
67 | movq %cr0, %rax |
68 | movq %rax, CR0(%r11) |
69 | movq %cr3, %rax |
70 | movq %rax, CR3(%r11) |
71 | movq %cr4, %rax |
72 | movq %rax, CR4(%r11) |
73 | |
74 | /* Save CR4. Required to enable the right paging mode later. */ |
75 | movq %rax, %r13 |
76 | |
77 | /* zero out flags, and disable interrupts */ |
78 | pushq $0 |
79 | popfq |
80 | |
81 | /* Save SME active flag */ |
82 | movq %r8, %r12 |
83 | |
84 | /* |
85 | * get physical address of control page now |
86 | * this is impossible after page table switch |
87 | */ |
88 | movq PTR(PA_CONTROL_PAGE)(%rsi), %r8 |
89 | |
90 | /* get physical address of page table now too */ |
91 | movq PTR(PA_TABLE_PAGE)(%rsi), %r9 |
92 | |
93 | /* get physical address of swap page now */ |
94 | movq PTR(PA_SWAP_PAGE)(%rsi), %r10 |
95 | |
96 | /* save some information for jumping back */ |
97 | movq %r9, CP_PA_TABLE_PAGE(%r11) |
98 | movq %r10, CP_PA_SWAP_PAGE(%r11) |
99 | movq %rdi, CP_PA_BACKUP_PAGES_MAP(%r11) |
100 | |
101 | /* Switch to the identity mapped page tables */ |
102 | movq %r9, %cr3 |
103 | |
104 | /* setup a new stack at the end of the physical control page */ |
105 | lea PAGE_SIZE(%r8), %rsp |
106 | |
107 | /* jump to identity mapped page */ |
108 | addq $(identity_mapped - relocate_kernel), %r8 |
109 | pushq %r8 |
110 | ANNOTATE_UNRET_SAFE |
111 | ret |
112 | int3 |
113 | SYM_CODE_END(relocate_kernel) |
114 | |
115 | SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) |
116 | UNWIND_HINT_END_OF_STACK |
117 | /* set return address to 0 if not preserving context */ |
118 | pushq $0 |
119 | /* store the start address on the stack */ |
120 | pushq %rdx |
121 | |
122 | /* |
123 | * Clear X86_CR4_CET (if it was set) such that we can clear CR0_WP |
124 | * below. |
125 | */ |
126 | movq %cr4, %rax |
127 | andq $~(X86_CR4_CET), %rax |
128 | movq %rax, %cr4 |
129 | |
130 | /* |
131 | * Set cr0 to a known state: |
132 | * - Paging enabled |
133 | * - Alignment check disabled |
134 | * - Write protect disabled |
135 | * - No task switch |
136 | * - Don't do FP software emulation. |
137 | * - Protected mode enabled |
138 | */ |
139 | movq %cr0, %rax |
140 | andq $~(X86_CR0_AM | X86_CR0_WP | X86_CR0_TS | X86_CR0_EM), %rax |
141 | orl $(X86_CR0_PG | X86_CR0_PE), %eax |
142 | movq %rax, %cr0 |
143 | |
144 | /* |
145 | * Set cr4 to a known state: |
146 | * - physical address extension enabled |
147 | * - 5-level paging, if it was enabled before |
148 | */ |
149 | movl $X86_CR4_PAE, %eax |
150 | testq $X86_CR4_LA57, %r13 |
151 | jz 1f |
152 | orl $X86_CR4_LA57, %eax |
153 | 1: |
154 | movq %rax, %cr4 |
155 | |
156 | jmp 1f |
157 | 1: |
158 | |
159 | /* Flush the TLB (needed?) */ |
160 | movq %r9, %cr3 |
161 | |
162 | /* |
163 | * If SME is active, there could be old encrypted cache line |
164 | * entries that will conflict with the now unencrypted memory |
165 | * used by kexec. Flush the caches before copying the kernel. |
166 | */ |
167 | testq %r12, %r12 |
168 | jz 1f |
169 | wbinvd |
170 | 1: |
171 | |
172 | movq %rcx, %r11 |
173 | call swap_pages |
174 | |
175 | /* |
176 | * To be certain of avoiding problems with self-modifying code |
177 | * I need to execute a serializing instruction here. |
178 | * So I flush the TLB by reloading %cr3 here, it's handy, |
179 | * and not processor dependent. |
180 | */ |
181 | movq %cr3, %rax |
182 | movq %rax, %cr3 |
183 | |
184 | /* |
185 | * set all of the registers to known values |
186 | * leave %rsp alone |
187 | */ |
188 | |
189 | testq %r11, %r11 |
190 | jnz 1f |
191 | xorl %eax, %eax |
192 | xorl %ebx, %ebx |
193 | xorl %ecx, %ecx |
194 | xorl %edx, %edx |
195 | xorl %esi, %esi |
196 | xorl %edi, %edi |
197 | xorl %ebp, %ebp |
198 | xorl %r8d, %r8d |
199 | xorl %r9d, %r9d |
200 | xorl %r10d, %r10d |
201 | xorl %r11d, %r11d |
202 | xorl %r12d, %r12d |
203 | xorl %r13d, %r13d |
204 | xorl %r14d, %r14d |
205 | xorl %r15d, %r15d |
206 | |
207 | ANNOTATE_UNRET_SAFE |
208 | ret |
209 | int3 |
210 | |
211 | 1: |
212 | popq %rdx |
213 | leaq PAGE_SIZE(%r10), %rsp |
214 | ANNOTATE_RETPOLINE_SAFE |
215 | call *%rdx |
216 | |
217 | /* get the re-entry point of the peer system */ |
218 | movq 0(%rsp), %rbp |
219 | leaq relocate_kernel(%rip), %r8 |
220 | movq CP_PA_SWAP_PAGE(%r8), %r10 |
221 | movq CP_PA_BACKUP_PAGES_MAP(%r8), %rdi |
222 | movq CP_PA_TABLE_PAGE(%r8), %rax |
223 | movq %rax, %cr3 |
224 | lea PAGE_SIZE(%r8), %rsp |
225 | call swap_pages |
226 | movq $virtual_mapped, %rax |
227 | pushq %rax |
228 | ANNOTATE_UNRET_SAFE |
229 | ret |
230 | int3 |
231 | SYM_CODE_END(identity_mapped) |
232 | |
233 | SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped) |
234 | UNWIND_HINT_END_OF_STACK |
235 | ANNOTATE_NOENDBR // RET target, above |
236 | movq RSP(%r8), %rsp |
237 | movq CR4(%r8), %rax |
238 | movq %rax, %cr4 |
239 | movq CR3(%r8), %rax |
240 | movq CR0(%r8), %r8 |
241 | movq %rax, %cr3 |
242 | movq %r8, %cr0 |
243 | movq %rbp, %rax |
244 | |
245 | popf |
246 | popq %r15 |
247 | popq %r14 |
248 | popq %r13 |
249 | popq %r12 |
250 | popq %rbp |
251 | popq %rbx |
252 | ANNOTATE_UNRET_SAFE |
253 | ret |
254 | int3 |
255 | SYM_CODE_END(virtual_mapped) |
256 | |
257 | /* Do the copies */ |
258 | SYM_CODE_START_LOCAL_NOALIGN(swap_pages) |
259 | UNWIND_HINT_END_OF_STACK |
260 | movq %rdi, %rcx /* Put the page_list in %rcx */ |
261 | xorl %edi, %edi |
262 | xorl %esi, %esi |
263 | jmp 1f |
264 | |
265 | 0: /* top, read another word for the indirection page */ |
266 | |
267 | movq (%rbx), %rcx |
268 | addq $8, %rbx |
269 | 1: |
270 | testb $0x1, %cl /* is it a destination page? */ |
271 | jz 2f |
272 | movq %rcx, %rdi |
273 | andq $0xfffffffffffff000, %rdi |
274 | jmp 0b |
275 | 2: |
276 | testb $0x2, %cl /* is it an indirection page? */ |
277 | jz 2f |
278 | movq %rcx, %rbx |
279 | andq $0xfffffffffffff000, %rbx |
280 | jmp 0b |
281 | 2: |
282 | testb $0x4, %cl /* is it the done indicator? */ |
283 | jz 2f |
284 | jmp 3f |
285 | 2: |
286 | testb $0x8, %cl /* is it the source indicator? */ |
287 | jz 0b /* Ignore it otherwise */ |
288 | movq %rcx, %rsi /* For ever source page do a copy */ |
289 | andq $0xfffffffffffff000, %rsi |
290 | |
291 | movq %rdi, %rdx |
292 | movq %rsi, %rax |
293 | |
294 | movq %r10, %rdi |
295 | movl $512, %ecx |
296 | rep ; movsq |
297 | |
298 | movq %rax, %rdi |
299 | movq %rdx, %rsi |
300 | movl $512, %ecx |
301 | rep ; movsq |
302 | |
303 | movq %rdx, %rdi |
304 | movq %r10, %rsi |
305 | movl $512, %ecx |
306 | rep ; movsq |
307 | |
308 | lea PAGE_SIZE(%rax), %rsi |
309 | jmp 0b |
310 | 3: |
311 | ANNOTATE_UNRET_SAFE |
312 | ret |
313 | int3 |
314 | SYM_CODE_END(swap_pages) |
315 | |
316 | .skip KEXEC_CONTROL_CODE_MAX_SIZE - (. - relocate_kernel), 0xcc |
317 | SYM_CODE_END(relocate_range); |
318 | |