1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Hibernation support for x86 |
4 | * |
5 | * Copyright (c) 2007 Rafael J. Wysocki <rjw@sisk.pl> |
6 | * Copyright (c) 2002 Pavel Machek <pavel@ucw.cz> |
7 | * Copyright (c) 2001 Patrick Mochel <mochel@osdl.org> |
8 | */ |
9 | #include <linux/gfp.h> |
10 | #include <linux/smp.h> |
11 | #include <linux/suspend.h> |
12 | #include <linux/scatterlist.h> |
13 | #include <linux/kdebug.h> |
14 | #include <linux/cpu.h> |
15 | #include <linux/pgtable.h> |
16 | #include <linux/types.h> |
17 | #include <linux/crc32.h> |
18 | |
19 | #include <asm/e820/api.h> |
20 | #include <asm/init.h> |
21 | #include <asm/proto.h> |
22 | #include <asm/page.h> |
23 | #include <asm/mtrr.h> |
24 | #include <asm/sections.h> |
25 | #include <asm/suspend.h> |
26 | #include <asm/tlbflush.h> |
27 | |
28 | /* |
29 | * Address to jump to in the last phase of restore in order to get to the image |
30 | * kernel's text (this value is passed in the image header). |
31 | */ |
32 | unsigned long restore_jump_address __visible; |
33 | unsigned long jump_address_phys; |
34 | |
35 | /* |
36 | * Value of the cr3 register from before the hibernation (this value is passed |
37 | * in the image header). |
38 | */ |
39 | unsigned long restore_cr3 __visible; |
40 | unsigned long temp_pgt __visible; |
41 | unsigned long relocated_restore_code __visible; |
42 | |
43 | /** |
44 | * pfn_is_nosave - check if given pfn is in the 'nosave' section |
45 | */ |
46 | int pfn_is_nosave(unsigned long pfn) |
47 | { |
48 | unsigned long nosave_begin_pfn; |
49 | unsigned long nosave_end_pfn; |
50 | |
51 | nosave_begin_pfn = __pa_symbol(&__nosave_begin) >> PAGE_SHIFT; |
52 | nosave_end_pfn = PAGE_ALIGN(__pa_symbol(&__nosave_end)) >> PAGE_SHIFT; |
53 | |
54 | return pfn >= nosave_begin_pfn && pfn < nosave_end_pfn; |
55 | } |
56 | |
57 | struct restore_data_record { |
58 | unsigned long jump_address; |
59 | unsigned long jump_address_phys; |
60 | unsigned long cr3; |
61 | unsigned long magic; |
62 | unsigned long e820_checksum; |
63 | }; |
64 | |
65 | /** |
66 | * compute_e820_crc32 - calculate crc32 of a given e820 table |
67 | * |
68 | * @table: the e820 table to be calculated |
69 | * |
70 | * Return: the resulting checksum |
71 | */ |
72 | static inline u32 compute_e820_crc32(struct e820_table *table) |
73 | { |
74 | int size = offsetof(struct e820_table, entries) + |
75 | sizeof(struct e820_entry) * table->nr_entries; |
76 | |
77 | return ~crc32_le(crc: ~0, p: (unsigned char const *)table, len: size); |
78 | } |
79 | |
80 | #ifdef CONFIG_X86_64 |
81 | #define RESTORE_MAGIC 0x23456789ABCDEF02UL |
82 | #else |
83 | #define RESTORE_MAGIC 0x12345679UL |
84 | #endif |
85 | |
86 | /** |
87 | * arch_hibernation_header_save - populate the architecture specific part |
88 | * of a hibernation image header |
89 | * @addr: address to save the data at |
90 | */ |
91 | int (void *addr, unsigned int max_size) |
92 | { |
93 | struct restore_data_record *rdr = addr; |
94 | |
95 | if (max_size < sizeof(struct restore_data_record)) |
96 | return -EOVERFLOW; |
97 | rdr->magic = RESTORE_MAGIC; |
98 | rdr->jump_address = (unsigned long)restore_registers; |
99 | rdr->jump_address_phys = __pa_symbol(restore_registers); |
100 | |
101 | /* |
102 | * The restore code fixes up CR3 and CR4 in the following sequence: |
103 | * |
104 | * [in hibernation asm] |
105 | * 1. CR3 <= temporary page tables |
106 | * 2. CR4 <= mmu_cr4_features (from the kernel that restores us) |
107 | * 3. CR3 <= rdr->cr3 |
108 | * 4. CR4 <= mmu_cr4_features (from us, i.e. the image kernel) |
109 | * [in restore_processor_state()] |
110 | * 5. CR4 <= saved CR4 |
111 | * 6. CR3 <= saved CR3 |
112 | * |
113 | * Our mmu_cr4_features has CR4.PCIDE=0, and toggling |
114 | * CR4.PCIDE while CR3's PCID bits are nonzero is illegal, so |
115 | * rdr->cr3 needs to point to valid page tables but must not |
116 | * have any of the PCID bits set. |
117 | */ |
118 | rdr->cr3 = restore_cr3 & ~CR3_PCID_MASK; |
119 | |
120 | rdr->e820_checksum = compute_e820_crc32(table: e820_table_firmware); |
121 | return 0; |
122 | } |
123 | |
124 | /** |
125 | * arch_hibernation_header_restore - read the architecture specific data |
126 | * from the hibernation image header |
127 | * @addr: address to read the data from |
128 | */ |
129 | int (void *addr) |
130 | { |
131 | struct restore_data_record *rdr = addr; |
132 | |
133 | if (rdr->magic != RESTORE_MAGIC) { |
134 | pr_crit("Unrecognized hibernate image header format!\n" ); |
135 | return -EINVAL; |
136 | } |
137 | |
138 | restore_jump_address = rdr->jump_address; |
139 | jump_address_phys = rdr->jump_address_phys; |
140 | restore_cr3 = rdr->cr3; |
141 | |
142 | if (rdr->e820_checksum != compute_e820_crc32(table: e820_table_firmware)) { |
143 | pr_crit("Hibernate inconsistent memory map detected!\n" ); |
144 | return -ENODEV; |
145 | } |
146 | |
147 | return 0; |
148 | } |
149 | |
150 | int relocate_restore_code(void) |
151 | { |
152 | pgd_t *pgd; |
153 | p4d_t *p4d; |
154 | pud_t *pud; |
155 | pmd_t *pmd; |
156 | pte_t *pte; |
157 | |
158 | relocated_restore_code = get_safe_page(GFP_ATOMIC); |
159 | if (!relocated_restore_code) |
160 | return -ENOMEM; |
161 | |
162 | __memcpy(to: (void *)relocated_restore_code, from: core_restore_code, PAGE_SIZE); |
163 | |
164 | /* Make the page containing the relocated code executable */ |
165 | pgd = (pgd_t *)__va(read_cr3_pa()) + |
166 | pgd_index(relocated_restore_code); |
167 | p4d = p4d_offset(pgd, address: relocated_restore_code); |
168 | if (p4d_leaf(p4d: *p4d)) { |
169 | set_p4d(p4dp: p4d, p4d: __p4d(val: p4d_val(p4d: *p4d) & ~_PAGE_NX)); |
170 | goto out; |
171 | } |
172 | pud = pud_offset(p4d, address: relocated_restore_code); |
173 | if (pud_leaf(pud: *pud)) { |
174 | set_pud(pudp: pud, pud: __pud(val: pud_val(pud: *pud) & ~_PAGE_NX)); |
175 | goto out; |
176 | } |
177 | pmd = pmd_offset(pud, address: relocated_restore_code); |
178 | if (pmd_leaf(pte: *pmd)) { |
179 | set_pmd(pmdp: pmd, pmd: __pmd(val: pmd_val(pmd: *pmd) & ~_PAGE_NX)); |
180 | goto out; |
181 | } |
182 | pte = pte_offset_kernel(pmd, address: relocated_restore_code); |
183 | set_pte(ptep: pte, pte: __pte(val: pte_val(pte: *pte) & ~_PAGE_NX)); |
184 | out: |
185 | __flush_tlb_all(); |
186 | return 0; |
187 | } |
188 | |
189 | int arch_resume_nosmt(void) |
190 | { |
191 | int ret = 0; |
192 | /* |
193 | * We reached this while coming out of hibernation. This means |
194 | * that SMT siblings are sleeping in hlt, as mwait is not safe |
195 | * against control transition during resume (see comment in |
196 | * hibernate_resume_nonboot_cpu_disable()). |
197 | * |
198 | * If the resumed kernel has SMT disabled, we have to take all the |
199 | * SMT siblings out of hlt, and offline them again so that they |
200 | * end up in mwait proper. |
201 | * |
202 | * Called with hotplug disabled. |
203 | */ |
204 | cpu_hotplug_enable(); |
205 | if (cpu_smt_control == CPU_SMT_DISABLED || |
206 | cpu_smt_control == CPU_SMT_FORCE_DISABLED) { |
207 | enum cpuhp_smt_control old = cpu_smt_control; |
208 | |
209 | ret = cpuhp_smt_enable(); |
210 | if (ret) |
211 | goto out; |
212 | ret = cpuhp_smt_disable(ctrlval: old); |
213 | if (ret) |
214 | goto out; |
215 | } |
216 | out: |
217 | cpu_hotplug_disable(); |
218 | return ret; |
219 | } |
220 | |