1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * This file contains kexec low-level functions.
4 *
5 * Copyright (C) 2002-2003 Eric Biederman <ebiederm@xmission.com>
6 * GameCube/ppc32 port Copyright (C) 2004 Albert Herranz
7 * PPC44x port. Copyright (C) 2011, IBM Corporation
8 * Author: Suzuki Poulose <suzuki@in.ibm.com>
9 */
10
11#include <linux/objtool.h>
12#include <asm/reg.h>
13#include <asm/page.h>
14#include <asm/mmu.h>
15#include <asm/ppc_asm.h>
16#include <asm/kexec.h>
17
18 .text
19
20 /*
21 * Must be relocatable PIC code callable as a C function.
22 */
23 .globl relocate_new_kernel
24relocate_new_kernel:
25 /* r3 = page_list */
26 /* r4 = reboot_code_buffer */
27 /* r5 = start_address */
28
29#ifdef CONFIG_PPC_85xx
30
31 mr r29, r3
32 mr r30, r4
33 mr r31, r5
34
35#define ENTRY_MAPPING_KEXEC_SETUP
36#include <kernel/85xx_entry_mapping.S>
37#undef ENTRY_MAPPING_KEXEC_SETUP
38
39 mr r3, r29
40 mr r4, r30
41 mr r5, r31
42
43 li r0, 0
44#elif defined(CONFIG_44x)
45
46 /* Save our parameters */
47 mr r29, r3
48 mr r30, r4
49 mr r31, r5
50
51#ifdef CONFIG_PPC_47x
52 /* Check for 47x cores */
53 mfspr r3,SPRN_PVR
54 srwi r3,r3,16
55 cmplwi cr0,r3,PVR_476FPE@h
56 beq setup_map_47x
57 cmplwi cr0,r3,PVR_476@h
58 beq setup_map_47x
59 cmplwi cr0,r3,PVR_476_ISS@h
60 beq setup_map_47x
61#endif /* CONFIG_PPC_47x */
62
63/*
64 * Code for setting up 1:1 mapping for PPC440x for KEXEC
65 *
66 * We cannot switch off the MMU on PPC44x.
67 * So we:
68 * 1) Invalidate all the mappings except the one we are running from.
69 * 2) Create a tmp mapping for our code in the other address space(TS) and
70 * jump to it. Invalidate the entry we started in.
71 * 3) Create a 1:1 mapping for 0-2GiB in chunks of 256M in original TS.
72 * 4) Jump to the 1:1 mapping in original TS.
73 * 5) Invalidate the tmp mapping.
74 *
75 * - Based on the kexec support code for FSL BookE
76 *
77 */
78
79 /*
80 * Load the PID with kernel PID (0).
81 * Also load our MSR_IS and TID to MMUCR for TLB search.
82 */
83 li r3, 0
84 mtspr SPRN_PID, r3
85 mfmsr r4
86 andi. r4,r4,MSR_IS@l
87 beq wmmucr
88 oris r3,r3,PPC44x_MMUCR_STS@h
89wmmucr:
90 mtspr SPRN_MMUCR,r3
91 sync
92
93 /*
94 * Invalidate all the TLB entries except the current entry
95 * where we are running from
96 */
97 bcl 20,31,$+4 /* Find our address */
980: mflr r5 /* Make it accessible */
99 tlbsx r23,0,r5 /* Find entry we are in */
100 li r4,0 /* Start at TLB entry 0 */
101 li r3,0 /* Set PAGEID inval value */
1021: cmpw r23,r4 /* Is this our entry? */
103 beq skip /* If so, skip the inval */
104 tlbwe r3,r4,PPC44x_TLB_PAGEID /* If not, inval the entry */
105skip:
106 addi r4,r4,1 /* Increment */
107 cmpwi r4,64 /* Are we done? */
108 bne 1b /* If not, repeat */
109 isync
110
111 /* Create a temp mapping and jump to it */
112 andi. r6, r23, 1 /* Find the index to use */
113 addi r24, r6, 1 /* r24 will contain 1 or 2 */
114
115 mfmsr r9 /* get the MSR */
116 rlwinm r5, r9, 27, 31, 31 /* Extract the MSR[IS] */
117 xori r7, r5, 1 /* Use the other address space */
118
119 /* Read the current mapping entries */
120 tlbre r3, r23, PPC44x_TLB_PAGEID
121 tlbre r4, r23, PPC44x_TLB_XLAT
122 tlbre r5, r23, PPC44x_TLB_ATTRIB
123
124 /* Save our current XLAT entry */
125 mr r25, r4
126
127 /* Extract the TLB PageSize */
128 li r10, 1 /* r10 will hold PageSize */
129 rlwinm r11, r3, 0, 24, 27 /* bits 24-27 */
130
131 /* XXX: As of now we use 256M, 4K pages */
132 cmpwi r11, PPC44x_TLB_256M
133 bne tlb_4k
134 rotlwi r10, r10, 28 /* r10 = 256M */
135 b write_out
136tlb_4k:
137 cmpwi r11, PPC44x_TLB_4K
138 bne default
139 rotlwi r10, r10, 12 /* r10 = 4K */
140 b write_out
141default:
142 rotlwi r10, r10, 10 /* r10 = 1K */
143
144write_out:
145 /*
146 * Write out the tmp 1:1 mapping for this code in other address space
147 * Fixup EPN = RPN , TS=other address space
148 */
149 insrwi r3, r7, 1, 23 /* Bit 23 is TS for PAGEID field */
150
151 /* Write out the tmp mapping entries */
152 tlbwe r3, r24, PPC44x_TLB_PAGEID
153 tlbwe r4, r24, PPC44x_TLB_XLAT
154 tlbwe r5, r24, PPC44x_TLB_ATTRIB
155
156 subi r11, r10, 1 /* PageOffset Mask = PageSize - 1 */
157 not r10, r11 /* Mask for PageNum */
158
159 /* Switch to other address space in MSR */
160 insrwi r9, r7, 1, 26 /* Set MSR[IS] = r7 */
161
162 bcl 20,31,$+4
1631: mflr r8
164 addi r8, r8, (2f-1b) /* Find the target offset */
165
166 /* Jump to the tmp mapping */
167 mtspr SPRN_SRR0, r8
168 mtspr SPRN_SRR1, r9
169 rfi
170
1712:
172 /* Invalidate the entry we were executing from */
173 li r3, 0
174 tlbwe r3, r23, PPC44x_TLB_PAGEID
175
176 /* attribute fields. rwx for SUPERVISOR mode */
177 li r5, 0
178 ori r5, r5, (PPC44x_TLB_SW | PPC44x_TLB_SR | PPC44x_TLB_SX | PPC44x_TLB_G)
179
180 /* Create 1:1 mapping in 256M pages */
181 xori r7, r7, 1 /* Revert back to Original TS */
182
183 li r8, 0 /* PageNumber */
184 li r6, 3 /* TLB Index, start at 3 */
185
186next_tlb:
187 rotlwi r3, r8, 28 /* Create EPN (bits 0-3) */
188 mr r4, r3 /* RPN = EPN */
189 ori r3, r3, (PPC44x_TLB_VALID | PPC44x_TLB_256M) /* SIZE = 256M, Valid */
190 insrwi r3, r7, 1, 23 /* Set TS from r7 */
191
192 tlbwe r3, r6, PPC44x_TLB_PAGEID /* PageID field : EPN, V, SIZE */
193 tlbwe r4, r6, PPC44x_TLB_XLAT /* Address translation : RPN */
194 tlbwe r5, r6, PPC44x_TLB_ATTRIB /* Attributes */
195
196 addi r8, r8, 1 /* Increment PN */
197 addi r6, r6, 1 /* Increment TLB Index */
198 cmpwi r8, 8 /* Are we done ? */
199 bne next_tlb
200 isync
201
202 /* Jump to the new mapping 1:1 */
203 li r9,0
204 insrwi r9, r7, 1, 26 /* Set MSR[IS] = r7 */
205
206 bcl 20,31,$+4
2071: mflr r8
208 and r8, r8, r11 /* Get our offset within page */
209 addi r8, r8, (2f-1b)
210
211 and r5, r25, r10 /* Get our target PageNum */
212 or r8, r8, r5 /* Target jump address */
213
214 mtspr SPRN_SRR0, r8
215 mtspr SPRN_SRR1, r9
216 rfi
2172:
218 /* Invalidate the tmp entry we used */
219 li r3, 0
220 tlbwe r3, r24, PPC44x_TLB_PAGEID
221 sync
222 b ppc44x_map_done
223
224#ifdef CONFIG_PPC_47x
225
226 /* 1:1 mapping for 47x */
227
228setup_map_47x:
229
230 /*
231 * Load the kernel pid (0) to PID and also to MMUCR[TID].
232 * Also set the MSR IS->MMUCR STS
233 */
234 li r3, 0
235 mtspr SPRN_PID, r3 /* Set PID */
236 mfmsr r4 /* Get MSR */
237 andi. r4, r4, MSR_IS@l /* TS=1? */
238 beq 1f /* If not, leave STS=0 */
239 oris r3, r3, PPC47x_MMUCR_STS@h /* Set STS=1 */
2401: mtspr SPRN_MMUCR, r3 /* Put MMUCR */
241 sync
242
243 /* Find the entry we are running from */
244 bcl 20,31,$+4
2452: mflr r23
246 tlbsx r23, 0, r23
247 tlbre r24, r23, 0 /* TLB Word 0 */
248 tlbre r25, r23, 1 /* TLB Word 1 */
249 tlbre r26, r23, 2 /* TLB Word 2 */
250
251
252 /*
253 * Invalidates all the tlb entries by writing to 256 RPNs(r4)
254 * of 4k page size in all 4 ways (0-3 in r3).
255 * This would invalidate the entire UTLB including the one we are
256 * running from. However the shadow TLB entries would help us
257 * to continue the execution, until we flush them (rfi/isync).
258 */
259 addis r3, 0, 0x8000 /* specify the way */
260 addi r4, 0, 0 /* TLB Word0 = (EPN=0, VALID = 0) */
261 addi r5, 0, 0
262 b clear_utlb_entry
263
264 /* Align the loop to speed things up. from head_44x.S */
265 .align 6
266
267clear_utlb_entry:
268
269 tlbwe r4, r3, 0
270 tlbwe r5, r3, 1
271 tlbwe r5, r3, 2
272 addis r3, r3, 0x2000 /* Increment the way */
273 cmpwi r3, 0
274 bne clear_utlb_entry
275 addis r3, 0, 0x8000
276 addis r4, r4, 0x100 /* Increment the EPN */
277 cmpwi r4, 0
278 bne clear_utlb_entry
279
280 /* Create the entries in the other address space */
281 mfmsr r5
282 rlwinm r7, r5, 27, 31, 31 /* Get the TS (Bit 26) from MSR */
283 xori r7, r7, 1 /* r7 = !TS */
284
285 insrwi r24, r7, 1, 21 /* Change the TS in the saved TLB word 0 */
286
287 /*
288 * write out the TLB entries for the tmp mapping
289 * Use way '0' so that we could easily invalidate it later.
290 */
291 lis r3, 0x8000 /* Way '0' */
292
293 tlbwe r24, r3, 0
294 tlbwe r25, r3, 1
295 tlbwe r26, r3, 2
296
297 /* Update the msr to the new TS */
298 insrwi r5, r7, 1, 26
299
300 bcl 20,31,$+4
3011: mflr r6
302 addi r6, r6, (2f-1b)
303
304 mtspr SPRN_SRR0, r6
305 mtspr SPRN_SRR1, r5
306 rfi
307
308 /*
309 * Now we are in the tmp address space.
310 * Create a 1:1 mapping for 0-2GiB in the original TS.
311 */
3122:
313 li r3, 0
314 li r4, 0 /* TLB Word 0 */
315 li r5, 0 /* TLB Word 1 */
316 li r6, 0
317 ori r6, r6, PPC47x_TLB2_S_RWX /* TLB word 2 */
318
319 li r8, 0 /* PageIndex */
320
321 xori r7, r7, 1 /* revert back to original TS */
322
323write_utlb:
324 rotlwi r5, r8, 28 /* RPN = PageIndex * 256M */
325 /* ERPN = 0 as we don't use memory above 2G */
326
327 mr r4, r5 /* EPN = RPN */
328 ori r4, r4, (PPC47x_TLB0_VALID | PPC47x_TLB0_256M)
329 insrwi r4, r7, 1, 21 /* Insert the TS to Word 0 */
330
331 tlbwe r4, r3, 0 /* Write out the entries */
332 tlbwe r5, r3, 1
333 tlbwe r6, r3, 2
334 addi r8, r8, 1
335 cmpwi r8, 8 /* Have we completed ? */
336 bne write_utlb
337
338 /* make sure we complete the TLB write up */
339 isync
340
341 /*
342 * Prepare to jump to the 1:1 mapping.
343 * 1) Extract page size of the tmp mapping
344 * DSIZ = TLB_Word0[22:27]
345 * 2) Calculate the physical address of the address
346 * to jump to.
347 */
348 rlwinm r10, r24, 0, 22, 27
349
350 cmpwi r10, PPC47x_TLB0_4K
351 bne 0f
352 li r10, 0x1000 /* r10 = 4k */
353 ANNOTATE_INTRA_FUNCTION_CALL
354 bl 1f
355
3560:
357 /* Defaults to 256M */
358 lis r10, 0x1000
359
360 bcl 20,31,$+4
3611: mflr r4
362 addi r4, r4, (2f-1b) /* virtual address of 2f */
363
364 subi r11, r10, 1 /* offsetmask = Pagesize - 1 */
365 not r10, r11 /* Pagemask = ~(offsetmask) */
366
367 and r5, r25, r10 /* Physical page */
368 and r6, r4, r11 /* offset within the current page */
369
370 or r5, r5, r6 /* Physical address for 2f */
371
372 /* Switch the TS in MSR to the original one */
373 mfmsr r8
374 insrwi r8, r7, 1, 26
375
376 mtspr SPRN_SRR1, r8
377 mtspr SPRN_SRR0, r5
378 rfi
379
3802:
381 /* Invalidate the tmp mapping */
382 lis r3, 0x8000 /* Way '0' */
383
384 clrrwi r24, r24, 12 /* Clear the valid bit */
385 tlbwe r24, r3, 0
386 tlbwe r25, r3, 1
387 tlbwe r26, r3, 2
388
389 /* Make sure we complete the TLB write and flush the shadow TLB */
390 isync
391
392#endif
393
394ppc44x_map_done:
395
396
397 /* Restore the parameters */
398 mr r3, r29
399 mr r4, r30
400 mr r5, r31
401
402 li r0, 0
403#else
404 li r0, 0
405
406 /*
407 * Set Machine Status Register to a known status,
408 * switch the MMU off and jump to 1: in a single step.
409 */
410
411 mr r8, r0
412 ori r8, r8, MSR_RI|MSR_ME
413 mtspr SPRN_SRR1, r8
414 addi r8, r4, 1f - relocate_new_kernel
415 mtspr SPRN_SRR0, r8
416 sync
417 rfi
418
4191:
420#endif
421 /* from this point address translation is turned off */
422 /* and interrupts are disabled */
423
424 /* set a new stack at the bottom of our page... */
425 /* (not really needed now) */
426 addi r1, r4, KEXEC_CONTROL_PAGE_SIZE - 8 /* for LR Save+Back Chain */
427 stw r0, 0(r1)
428
429 /* Do the copies */
430 li r6, 0 /* checksum */
431 mr r0, r3
432 b 1f
433
4340: /* top, read another word for the indirection page */
435 lwzu r0, 4(r3)
436
4371:
438 /* is it a destination page? (r8) */
439 rlwinm. r7, r0, 0, 31, 31 /* IND_DESTINATION (1<<0) */
440 beq 2f
441
442 rlwinm r8, r0, 0, 0, 19 /* clear kexec flags, page align */
443 b 0b
444
4452: /* is it an indirection page? (r3) */
446 rlwinm. r7, r0, 0, 30, 30 /* IND_INDIRECTION (1<<1) */
447 beq 2f
448
449 rlwinm r3, r0, 0, 0, 19 /* clear kexec flags, page align */
450 subi r3, r3, 4
451 b 0b
452
4532: /* are we done? */
454 rlwinm. r7, r0, 0, 29, 29 /* IND_DONE (1<<2) */
455 beq 2f
456 b 3f
457
4582: /* is it a source page? (r9) */
459 rlwinm. r7, r0, 0, 28, 28 /* IND_SOURCE (1<<3) */
460 beq 0b
461
462 rlwinm r9, r0, 0, 0, 19 /* clear kexec flags, page align */
463
464 li r7, PAGE_SIZE / 4
465 mtctr r7
466 subi r9, r9, 4
467 subi r8, r8, 4
4689:
469 lwzu r0, 4(r9) /* do the copy */
470 xor r6, r6, r0
471 stwu r0, 4(r8)
472 dcbst 0, r8
473 sync
474 icbi 0, r8
475 bdnz 9b
476
477 addi r9, r9, 4
478 addi r8, r8, 4
479 b 0b
480
4813:
482
483 /* To be certain of avoiding problems with self-modifying code
484 * execute a serializing instruction here.
485 */
486 isync
487 sync
488
489 mfspr r3, SPRN_PIR /* current core we are running on */
490 mr r4, r5 /* load physical address of chunk called */
491
492 /* jump to the entry point, usually the setup routine */
493 mtlr r5
494 blrl
495
4961: b 1b
497
498relocate_new_kernel_end:
499
500 .globl relocate_new_kernel_size
501relocate_new_kernel_size:
502 .long relocate_new_kernel_end - relocate_new_kernel
503

source code of linux/arch/powerpc/kexec/relocate_32.S