1 | /* |
2 | * Helper macros to support writing architecture specific |
3 | * linker scripts. |
4 | * |
5 | * A minimal linker scripts has following content: |
6 | * [This is a sample, architectures may have special requiriements] |
7 | * |
8 | * OUTPUT_FORMAT(...) |
9 | * OUTPUT_ARCH(...) |
10 | * ENTRY(...) |
11 | * SECTIONS |
12 | * { |
13 | * . = START; |
14 | * __init_begin = .; |
15 | * HEAD_TEXT_SECTION |
16 | * INIT_TEXT_SECTION(PAGE_SIZE) |
17 | * INIT_DATA_SECTION(...) |
18 | * PERCPU_SECTION(CACHELINE_SIZE) |
19 | * __init_end = .; |
20 | * |
21 | * _stext = .; |
22 | * TEXT_SECTION = 0 |
23 | * _etext = .; |
24 | * |
25 | * _sdata = .; |
26 | * RO_DATA(PAGE_SIZE) |
27 | * RW_DATA(...) |
28 | * _edata = .; |
29 | * |
30 | * EXCEPTION_TABLE(...) |
31 | * |
32 | * BSS_SECTION(0, 0, 0) |
33 | * _end = .; |
34 | * |
35 | * STABS_DEBUG |
36 | * DWARF_DEBUG |
37 | * ELF_DETAILS |
38 | * |
39 | * DISCARDS // must be the last |
40 | * } |
41 | * |
42 | * [__init_begin, __init_end] is the init section that may be freed after init |
43 | * // __init_begin and __init_end should be page aligned, so that we can |
44 | * // free the whole .init memory |
45 | * [_stext, _etext] is the text section |
46 | * [_sdata, _edata] is the data section |
47 | * |
48 | * Some of the included output section have their own set of constants. |
49 | * Examples are: [__initramfs_start, __initramfs_end] for initramfs and |
50 | * [__nosave_begin, __nosave_end] for the nosave data |
51 | */ |
52 | |
53 | #ifndef LOAD_OFFSET |
54 | #define LOAD_OFFSET 0 |
55 | #endif |
56 | |
57 | /* |
58 | * Only some architectures want to have the .notes segment visible in |
59 | * a separate PT_NOTE ELF Program Header. When this happens, it needs |
60 | * to be visible in both the kernel text's PT_LOAD and the PT_NOTE |
61 | * Program Headers. In this case, though, the PT_LOAD needs to be made |
62 | * the default again so that all the following sections don't also end |
63 | * up in the PT_NOTE Program Header. |
64 | */ |
65 | #ifdef EMITS_PT_NOTE |
66 | #define :text :note |
67 | #define __restore_ph : { *(.__restore_ph) } :text |
68 | #else |
69 | #define NOTES_HEADERS |
70 | #define NOTES_HEADERS_RESTORE |
71 | #endif |
72 | |
73 | /* |
74 | * Some architectures have non-executable read-only exception tables. |
75 | * They can be added to the RO_DATA segment by specifying their desired |
76 | * alignment. |
77 | */ |
78 | #ifdef RO_EXCEPTION_TABLE_ALIGN |
79 | #define RO_EXCEPTION_TABLE EXCEPTION_TABLE(RO_EXCEPTION_TABLE_ALIGN) |
80 | #else |
81 | #define RO_EXCEPTION_TABLE |
82 | #endif |
83 | |
84 | /* Align . function alignment. */ |
85 | #define ALIGN_FUNCTION() . = ALIGN(CONFIG_FUNCTION_ALIGNMENT) |
86 | |
87 | /* |
88 | * LD_DEAD_CODE_DATA_ELIMINATION option enables -fdata-sections, which |
89 | * generates .data.identifier sections, which need to be pulled in with |
90 | * .data. We don't want to pull in .data..other sections, which Linux |
91 | * has defined. Same for text and bss. |
92 | * |
93 | * With LTO_CLANG, the linker also splits sections by default, so we need |
94 | * these macros to combine the sections during the final link. |
95 | * |
96 | * RODATA_MAIN is not used because existing code already defines .rodata.x |
97 | * sections to be brought in with rodata. |
98 | */ |
99 | #if defined(CONFIG_LD_DEAD_CODE_DATA_ELIMINATION) || defined(CONFIG_LTO_CLANG) |
100 | #define TEXT_MAIN .text .text.[0-9a-zA-Z_]* |
101 | #define DATA_MAIN .data .data.[0-9a-zA-Z_]* .data..L* .data..compoundliteral* .data.$__unnamed_* .data.$L* |
102 | #define SDATA_MAIN .sdata .sdata.[0-9a-zA-Z_]* |
103 | #define RODATA_MAIN .rodata .rodata.[0-9a-zA-Z_]* .rodata..L* |
104 | #define BSS_MAIN .bss .bss.[0-9a-zA-Z_]* .bss..compoundliteral* |
105 | #define SBSS_MAIN .sbss .sbss.[0-9a-zA-Z_]* |
106 | #else |
107 | #define TEXT_MAIN .text |
108 | #define DATA_MAIN .data |
109 | #define SDATA_MAIN .sdata |
110 | #define RODATA_MAIN .rodata |
111 | #define BSS_MAIN .bss |
112 | #define SBSS_MAIN .sbss |
113 | #endif |
114 | |
115 | /* |
116 | * GCC 4.5 and later have a 32 bytes section alignment for structures. |
117 | * Except GCC 4.9, that feels the need to align on 64 bytes. |
118 | */ |
119 | #define STRUCT_ALIGNMENT 32 |
120 | #define STRUCT_ALIGN() . = ALIGN(STRUCT_ALIGNMENT) |
121 | |
122 | /* |
123 | * The order of the sched class addresses are important, as they are |
124 | * used to determine the order of the priority of each sched class in |
125 | * relation to each other. |
126 | */ |
127 | #define SCHED_DATA \ |
128 | STRUCT_ALIGN(); \ |
129 | __sched_class_highest = .; \ |
130 | *(__stop_sched_class) \ |
131 | *(__dl_sched_class) \ |
132 | *(__rt_sched_class) \ |
133 | *(__fair_sched_class) \ |
134 | *(__idle_sched_class) \ |
135 | __sched_class_lowest = .; |
136 | |
137 | /* The actual configuration determine if the init/exit sections |
138 | * are handled as text/data or they can be discarded (which |
139 | * often happens at runtime) |
140 | */ |
141 | |
142 | #if defined(CONFIG_MEMORY_HOTPLUG) |
143 | #define MEM_KEEP(sec) *(.mem##sec) |
144 | #define MEM_DISCARD(sec) |
145 | #else |
146 | #define MEM_KEEP(sec) |
147 | #define MEM_DISCARD(sec) *(.mem##sec) |
148 | #endif |
149 | |
150 | #ifndef CONFIG_HAVE_DYNAMIC_FTRACE_NO_PATCHABLE |
151 | #define KEEP_PATCHABLE KEEP(*(__patchable_function_entries)) |
152 | #define PATCHABLE_DISCARDS |
153 | #else |
154 | #define KEEP_PATCHABLE |
155 | #define PATCHABLE_DISCARDS *(__patchable_function_entries) |
156 | #endif |
157 | |
158 | #ifndef CONFIG_ARCH_SUPPORTS_CFI_CLANG |
159 | /* |
160 | * Simply points to ftrace_stub, but with the proper protocol. |
161 | * Defined by the linker script in linux/vmlinux.lds.h |
162 | */ |
163 | #define FTRACE_STUB_HACK ftrace_stub_graph = ftrace_stub; |
164 | #else |
165 | #define FTRACE_STUB_HACK |
166 | #endif |
167 | |
168 | #ifdef CONFIG_FTRACE_MCOUNT_RECORD |
169 | /* |
170 | * The ftrace call sites are logged to a section whose name depends on the |
171 | * compiler option used. A given kernel image will only use one, AKA |
172 | * FTRACE_CALLSITE_SECTION. We capture all of them here to avoid header |
173 | * dependencies for FTRACE_CALLSITE_SECTION's definition. |
174 | * |
175 | * ftrace_ops_list_func will be defined as arch_ftrace_ops_list_func |
176 | * as some archs will have a different prototype for that function |
177 | * but ftrace_ops_list_func() will have a single prototype. |
178 | */ |
179 | #define MCOUNT_REC() . = ALIGN(8); \ |
180 | __start_mcount_loc = .; \ |
181 | KEEP(*(__mcount_loc)) \ |
182 | KEEP_PATCHABLE \ |
183 | __stop_mcount_loc = .; \ |
184 | FTRACE_STUB_HACK \ |
185 | ftrace_ops_list_func = arch_ftrace_ops_list_func; |
186 | #else |
187 | # ifdef CONFIG_FUNCTION_TRACER |
188 | # define MCOUNT_REC() FTRACE_STUB_HACK \ |
189 | ftrace_ops_list_func = arch_ftrace_ops_list_func; |
190 | # else |
191 | # define MCOUNT_REC() |
192 | # endif |
193 | #endif |
194 | |
195 | #define BOUNDED_SECTION_PRE_LABEL(_sec_, _label_, _BEGIN_, _END_) \ |
196 | _BEGIN_##_label_ = .; \ |
197 | KEEP(*(_sec_)) \ |
198 | _END_##_label_ = .; |
199 | |
200 | #define BOUNDED_SECTION_POST_LABEL(_sec_, _label_, _BEGIN_, _END_) \ |
201 | _label_##_BEGIN_ = .; \ |
202 | KEEP(*(_sec_)) \ |
203 | _label_##_END_ = .; |
204 | |
205 | #define BOUNDED_SECTION_BY(_sec_, _label_) \ |
206 | BOUNDED_SECTION_PRE_LABEL(_sec_, _label_, __start, __stop) |
207 | |
208 | #define BOUNDED_SECTION(_sec) BOUNDED_SECTION_BY(_sec, _sec) |
209 | |
210 | #define (_sec_, _label_, _BEGIN_, _END_, _HDR_) \ |
211 | _HDR_##_label_ = .; \ |
212 | KEEP(*(.gnu.linkonce.##_sec_)) \ |
213 | BOUNDED_SECTION_PRE_LABEL(_sec_, _label_, _BEGIN_, _END_) |
214 | |
215 | #define (_sec_, _label_, _BEGIN_, _END_, _HDR_) \ |
216 | _label_##_HDR_ = .; \ |
217 | KEEP(*(.gnu.linkonce.##_sec_)) \ |
218 | BOUNDED_SECTION_POST_LABEL(_sec_, _label_, _BEGIN_, _END_) |
219 | |
220 | #define (_sec_, _label_) \ |
221 | HEADERED_SECTION_PRE_LABEL(_sec_, _label_, __start, __stop) |
222 | |
223 | #define (_sec) HEADERED_SECTION_BY(_sec, _sec) |
224 | |
225 | #ifdef CONFIG_TRACE_BRANCH_PROFILING |
226 | #define LIKELY_PROFILE() \ |
227 | BOUNDED_SECTION_BY(_ftrace_annotated_branch, _annotated_branch_profile) |
228 | #else |
229 | #define LIKELY_PROFILE() |
230 | #endif |
231 | |
232 | #ifdef CONFIG_PROFILE_ALL_BRANCHES |
233 | #define BRANCH_PROFILE() \ |
234 | BOUNDED_SECTION_BY(_ftrace_branch, _branch_profile) |
235 | #else |
236 | #define BRANCH_PROFILE() |
237 | #endif |
238 | |
239 | #ifdef CONFIG_KPROBES |
240 | #define KPROBE_BLACKLIST() \ |
241 | . = ALIGN(8); \ |
242 | BOUNDED_SECTION(_kprobe_blacklist) |
243 | #else |
244 | #define KPROBE_BLACKLIST() |
245 | #endif |
246 | |
247 | #ifdef CONFIG_FUNCTION_ERROR_INJECTION |
248 | #define ERROR_INJECT_WHITELIST() \ |
249 | STRUCT_ALIGN(); \ |
250 | BOUNDED_SECTION(_error_injection_whitelist) |
251 | #else |
252 | #define ERROR_INJECT_WHITELIST() |
253 | #endif |
254 | |
255 | #ifdef CONFIG_EVENT_TRACING |
256 | #define FTRACE_EVENTS() \ |
257 | . = ALIGN(8); \ |
258 | BOUNDED_SECTION(_ftrace_events) \ |
259 | BOUNDED_SECTION_BY(_ftrace_eval_map, _ftrace_eval_maps) |
260 | #else |
261 | #define FTRACE_EVENTS() |
262 | #endif |
263 | |
264 | #ifdef CONFIG_TRACING |
265 | #define TRACE_PRINTKS() BOUNDED_SECTION_BY(__trace_printk_fmt, ___trace_bprintk_fmt) |
266 | #define TRACEPOINT_STR() BOUNDED_SECTION_BY(__tracepoint_str, ___tracepoint_str) |
267 | #else |
268 | #define TRACE_PRINTKS() |
269 | #define TRACEPOINT_STR() |
270 | #endif |
271 | |
272 | #ifdef CONFIG_FTRACE_SYSCALLS |
273 | #define TRACE_SYSCALLS() \ |
274 | . = ALIGN(8); \ |
275 | BOUNDED_SECTION_BY(__syscalls_metadata, _syscalls_metadata) |
276 | #else |
277 | #define TRACE_SYSCALLS() |
278 | #endif |
279 | |
280 | #ifdef CONFIG_BPF_EVENTS |
281 | #define BPF_RAW_TP() STRUCT_ALIGN(); \ |
282 | BOUNDED_SECTION_BY(__bpf_raw_tp_map, __bpf_raw_tp) |
283 | #else |
284 | #define BPF_RAW_TP() |
285 | #endif |
286 | |
287 | #ifdef CONFIG_SERIAL_EARLYCON |
288 | #define EARLYCON_TABLE() \ |
289 | . = ALIGN(8); \ |
290 | BOUNDED_SECTION_POST_LABEL(__earlycon_table, __earlycon_table, , _end) |
291 | #else |
292 | #define EARLYCON_TABLE() |
293 | #endif |
294 | |
295 | #ifdef CONFIG_SECURITY |
296 | #define LSM_TABLE() \ |
297 | . = ALIGN(8); \ |
298 | BOUNDED_SECTION_PRE_LABEL(.lsm_info.init, _lsm_info, __start, __end) |
299 | |
300 | #define EARLY_LSM_TABLE() \ |
301 | . = ALIGN(8); \ |
302 | BOUNDED_SECTION_PRE_LABEL(.early_lsm_info.init, _early_lsm_info, __start, __end) |
303 | #else |
304 | #define LSM_TABLE() |
305 | #define EARLY_LSM_TABLE() |
306 | #endif |
307 | |
308 | #define ___OF_TABLE(cfg, name) _OF_TABLE_##cfg(name) |
309 | #define __OF_TABLE(cfg, name) ___OF_TABLE(cfg, name) |
310 | #define OF_TABLE(cfg, name) __OF_TABLE(IS_ENABLED(cfg), name) |
311 | #define _OF_TABLE_0(name) |
312 | #define _OF_TABLE_1(name) \ |
313 | . = ALIGN(8); \ |
314 | __##name##_of_table = .; \ |
315 | KEEP(*(__##name##_of_table)) \ |
316 | KEEP(*(__##name##_of_table_end)) |
317 | |
318 | #define TIMER_OF_TABLES() OF_TABLE(CONFIG_TIMER_OF, timer) |
319 | #define IRQCHIP_OF_MATCH_TABLE() OF_TABLE(CONFIG_IRQCHIP, irqchip) |
320 | #define CLK_OF_TABLES() OF_TABLE(CONFIG_COMMON_CLK, clk) |
321 | #define RESERVEDMEM_OF_TABLES() OF_TABLE(CONFIG_OF_RESERVED_MEM, reservedmem) |
322 | #define CPU_METHOD_OF_TABLES() OF_TABLE(CONFIG_SMP, cpu_method) |
323 | #define CPUIDLE_METHOD_OF_TABLES() OF_TABLE(CONFIG_CPU_IDLE, cpuidle_method) |
324 | |
325 | #ifdef CONFIG_ACPI |
326 | #define ACPI_PROBE_TABLE(name) \ |
327 | . = ALIGN(8); \ |
328 | BOUNDED_SECTION_POST_LABEL(__##name##_acpi_probe_table, \ |
329 | __##name##_acpi_probe_table,, _end) |
330 | #else |
331 | #define ACPI_PROBE_TABLE(name) |
332 | #endif |
333 | |
334 | #ifdef CONFIG_THERMAL |
335 | #define THERMAL_TABLE(name) \ |
336 | . = ALIGN(8); \ |
337 | BOUNDED_SECTION_POST_LABEL(__##name##_thermal_table, \ |
338 | __##name##_thermal_table,, _end) |
339 | #else |
340 | #define THERMAL_TABLE(name) |
341 | #endif |
342 | |
343 | #define KERNEL_DTB() \ |
344 | STRUCT_ALIGN(); \ |
345 | __dtb_start = .; \ |
346 | KEEP(*(.dtb.init.rodata)) \ |
347 | __dtb_end = .; |
348 | |
349 | /* |
350 | * .data section |
351 | */ |
352 | #define DATA_DATA \ |
353 | *(.xiptext) \ |
354 | *(DATA_MAIN) \ |
355 | *(.data..decrypted) \ |
356 | *(.ref.data) \ |
357 | *(.data..shared_aligned) /* percpu related */ \ |
358 | MEM_KEEP(init.data*) \ |
359 | *(.data.unlikely) \ |
360 | __start_once = .; \ |
361 | *(.data.once) \ |
362 | __end_once = .; \ |
363 | STRUCT_ALIGN(); \ |
364 | *(__tracepoints) \ |
365 | /* implement dynamic printk debug */ \ |
366 | . = ALIGN(8); \ |
367 | BOUNDED_SECTION_BY(__dyndbg_classes, ___dyndbg_classes) \ |
368 | BOUNDED_SECTION_BY(__dyndbg, ___dyndbg) \ |
369 | LIKELY_PROFILE() \ |
370 | BRANCH_PROFILE() \ |
371 | TRACE_PRINTKS() \ |
372 | BPF_RAW_TP() \ |
373 | TRACEPOINT_STR() \ |
374 | KUNIT_TABLE() |
375 | |
376 | /* |
377 | * Data section helpers |
378 | */ |
379 | #define NOSAVE_DATA \ |
380 | . = ALIGN(PAGE_SIZE); \ |
381 | __nosave_begin = .; \ |
382 | *(.data..nosave) \ |
383 | . = ALIGN(PAGE_SIZE); \ |
384 | __nosave_end = .; |
385 | |
386 | #define PAGE_ALIGNED_DATA(page_align) \ |
387 | . = ALIGN(page_align); \ |
388 | *(.data..page_aligned) \ |
389 | . = ALIGN(page_align); |
390 | |
391 | #define READ_MOSTLY_DATA(align) \ |
392 | . = ALIGN(align); \ |
393 | *(.data..read_mostly) \ |
394 | . = ALIGN(align); |
395 | |
396 | #define CACHELINE_ALIGNED_DATA(align) \ |
397 | . = ALIGN(align); \ |
398 | *(.data..cacheline_aligned) |
399 | |
400 | #define INIT_TASK_DATA(align) \ |
401 | . = ALIGN(align); \ |
402 | __start_init_task = .; \ |
403 | init_thread_union = .; \ |
404 | init_stack = .; \ |
405 | KEEP(*(.data..init_task)) \ |
406 | KEEP(*(.data..init_thread_info)) \ |
407 | . = __start_init_task + THREAD_SIZE; \ |
408 | __end_init_task = .; |
409 | |
410 | #define JUMP_TABLE_DATA \ |
411 | . = ALIGN(8); \ |
412 | BOUNDED_SECTION_BY(__jump_table, ___jump_table) |
413 | |
414 | #ifdef CONFIG_HAVE_STATIC_CALL_INLINE |
415 | #define STATIC_CALL_DATA \ |
416 | . = ALIGN(8); \ |
417 | BOUNDED_SECTION_BY(.static_call_sites, _static_call_sites) \ |
418 | BOUNDED_SECTION_BY(.static_call_tramp_key, _static_call_tramp_key) |
419 | #else |
420 | #define STATIC_CALL_DATA |
421 | #endif |
422 | |
423 | /* |
424 | * Allow architectures to handle ro_after_init data on their |
425 | * own by defining an empty RO_AFTER_INIT_DATA. |
426 | */ |
427 | #ifndef RO_AFTER_INIT_DATA |
428 | #define RO_AFTER_INIT_DATA \ |
429 | . = ALIGN(8); \ |
430 | __start_ro_after_init = .; \ |
431 | *(.data..ro_after_init) \ |
432 | JUMP_TABLE_DATA \ |
433 | STATIC_CALL_DATA \ |
434 | __end_ro_after_init = .; |
435 | #endif |
436 | |
437 | /* |
438 | * .kcfi_traps contains a list KCFI trap locations. |
439 | */ |
440 | #ifndef KCFI_TRAPS |
441 | #ifdef CONFIG_ARCH_USES_CFI_TRAPS |
442 | #define KCFI_TRAPS \ |
443 | __kcfi_traps : AT(ADDR(__kcfi_traps) - LOAD_OFFSET) { \ |
444 | BOUNDED_SECTION_BY(.kcfi_traps, ___kcfi_traps) \ |
445 | } |
446 | #else |
447 | #define KCFI_TRAPS |
448 | #endif |
449 | #endif |
450 | |
451 | /* |
452 | * Read only Data |
453 | */ |
454 | #define RO_DATA(align) \ |
455 | . = ALIGN((align)); \ |
456 | .rodata : AT(ADDR(.rodata) - LOAD_OFFSET) { \ |
457 | __start_rodata = .; \ |
458 | *(.rodata) *(.rodata.*) \ |
459 | SCHED_DATA \ |
460 | RO_AFTER_INIT_DATA /* Read only after init */ \ |
461 | . = ALIGN(8); \ |
462 | BOUNDED_SECTION_BY(__tracepoints_ptrs, ___tracepoints_ptrs) \ |
463 | *(__tracepoints_strings)/* Tracepoints: strings */ \ |
464 | } \ |
465 | \ |
466 | .rodata1 : AT(ADDR(.rodata1) - LOAD_OFFSET) { \ |
467 | *(.rodata1) \ |
468 | } \ |
469 | \ |
470 | /* PCI quirks */ \ |
471 | .pci_fixup : AT(ADDR(.pci_fixup) - LOAD_OFFSET) { \ |
472 | BOUNDED_SECTION_PRE_LABEL(.pci_fixup_early, _pci_fixups_early, __start, __end) \ |
473 | BOUNDED_SECTION_PRE_LABEL(.pci_fixup_header, _pci_fixups_header, __start, __end) \ |
474 | BOUNDED_SECTION_PRE_LABEL(.pci_fixup_final, _pci_fixups_final, __start, __end) \ |
475 | BOUNDED_SECTION_PRE_LABEL(.pci_fixup_enable, _pci_fixups_enable, __start, __end) \ |
476 | BOUNDED_SECTION_PRE_LABEL(.pci_fixup_resume, _pci_fixups_resume, __start, __end) \ |
477 | BOUNDED_SECTION_PRE_LABEL(.pci_fixup_suspend, _pci_fixups_suspend, __start, __end) \ |
478 | BOUNDED_SECTION_PRE_LABEL(.pci_fixup_resume_early, _pci_fixups_resume_early, __start, __end) \ |
479 | BOUNDED_SECTION_PRE_LABEL(.pci_fixup_suspend_late, _pci_fixups_suspend_late, __start, __end) \ |
480 | } \ |
481 | \ |
482 | FW_LOADER_BUILT_IN_DATA \ |
483 | TRACEDATA \ |
484 | \ |
485 | PRINTK_INDEX \ |
486 | \ |
487 | /* Kernel symbol table: Normal symbols */ \ |
488 | __ksymtab : AT(ADDR(__ksymtab) - LOAD_OFFSET) { \ |
489 | __start___ksymtab = .; \ |
490 | KEEP(*(SORT(___ksymtab+*))) \ |
491 | __stop___ksymtab = .; \ |
492 | } \ |
493 | \ |
494 | /* Kernel symbol table: GPL-only symbols */ \ |
495 | __ksymtab_gpl : AT(ADDR(__ksymtab_gpl) - LOAD_OFFSET) { \ |
496 | __start___ksymtab_gpl = .; \ |
497 | KEEP(*(SORT(___ksymtab_gpl+*))) \ |
498 | __stop___ksymtab_gpl = .; \ |
499 | } \ |
500 | \ |
501 | /* Kernel symbol table: Normal symbols */ \ |
502 | __kcrctab : AT(ADDR(__kcrctab) - LOAD_OFFSET) { \ |
503 | __start___kcrctab = .; \ |
504 | KEEP(*(SORT(___kcrctab+*))) \ |
505 | __stop___kcrctab = .; \ |
506 | } \ |
507 | \ |
508 | /* Kernel symbol table: GPL-only symbols */ \ |
509 | __kcrctab_gpl : AT(ADDR(__kcrctab_gpl) - LOAD_OFFSET) { \ |
510 | __start___kcrctab_gpl = .; \ |
511 | KEEP(*(SORT(___kcrctab_gpl+*))) \ |
512 | __stop___kcrctab_gpl = .; \ |
513 | } \ |
514 | \ |
515 | /* Kernel symbol table: strings */ \ |
516 | __ksymtab_strings : AT(ADDR(__ksymtab_strings) - LOAD_OFFSET) { \ |
517 | *(__ksymtab_strings) \ |
518 | } \ |
519 | \ |
520 | /* __*init sections */ \ |
521 | __init_rodata : AT(ADDR(__init_rodata) - LOAD_OFFSET) { \ |
522 | *(.ref.rodata) \ |
523 | MEM_KEEP(init.rodata) \ |
524 | } \ |
525 | \ |
526 | /* Built-in module parameters. */ \ |
527 | __param : AT(ADDR(__param) - LOAD_OFFSET) { \ |
528 | BOUNDED_SECTION_BY(__param, ___param) \ |
529 | } \ |
530 | \ |
531 | /* Built-in module versions. */ \ |
532 | __modver : AT(ADDR(__modver) - LOAD_OFFSET) { \ |
533 | BOUNDED_SECTION_BY(__modver, ___modver) \ |
534 | } \ |
535 | \ |
536 | KCFI_TRAPS \ |
537 | \ |
538 | RO_EXCEPTION_TABLE \ |
539 | NOTES \ |
540 | BTF \ |
541 | \ |
542 | . = ALIGN((align)); \ |
543 | __end_rodata = .; |
544 | |
545 | |
546 | /* |
547 | * Non-instrumentable text section |
548 | */ |
549 | #define NOINSTR_TEXT \ |
550 | ALIGN_FUNCTION(); \ |
551 | __noinstr_text_start = .; \ |
552 | *(.noinstr.text) \ |
553 | __cpuidle_text_start = .; \ |
554 | *(.cpuidle.text) \ |
555 | __cpuidle_text_end = .; \ |
556 | __noinstr_text_end = .; |
557 | |
558 | /* |
559 | * .text section. Map to function alignment to avoid address changes |
560 | * during second ld run in second ld pass when generating System.map |
561 | * |
562 | * TEXT_MAIN here will match .text.fixup and .text.unlikely if dead |
563 | * code elimination is enabled, so these sections should be converted |
564 | * to use ".." first. |
565 | */ |
566 | #define TEXT_TEXT \ |
567 | ALIGN_FUNCTION(); \ |
568 | *(.text.hot .text.hot.*) \ |
569 | *(TEXT_MAIN .text.fixup) \ |
570 | *(.text.unlikely .text.unlikely.*) \ |
571 | *(.text.unknown .text.unknown.*) \ |
572 | NOINSTR_TEXT \ |
573 | *(.ref.text) \ |
574 | *(.text.asan.* .text.tsan.*) \ |
575 | MEM_KEEP(init.text*) \ |
576 | |
577 | |
578 | /* sched.text is aling to function alignment to secure we have same |
579 | * address even at second ld pass when generating System.map */ |
580 | #define SCHED_TEXT \ |
581 | ALIGN_FUNCTION(); \ |
582 | __sched_text_start = .; \ |
583 | *(.sched.text) \ |
584 | __sched_text_end = .; |
585 | |
586 | /* spinlock.text is aling to function alignment to secure we have same |
587 | * address even at second ld pass when generating System.map */ |
588 | #define LOCK_TEXT \ |
589 | ALIGN_FUNCTION(); \ |
590 | __lock_text_start = .; \ |
591 | *(.spinlock.text) \ |
592 | __lock_text_end = .; |
593 | |
594 | #define KPROBES_TEXT \ |
595 | ALIGN_FUNCTION(); \ |
596 | __kprobes_text_start = .; \ |
597 | *(.kprobes.text) \ |
598 | __kprobes_text_end = .; |
599 | |
600 | #define ENTRY_TEXT \ |
601 | ALIGN_FUNCTION(); \ |
602 | __entry_text_start = .; \ |
603 | *(.entry.text) \ |
604 | __entry_text_end = .; |
605 | |
606 | #define IRQENTRY_TEXT \ |
607 | ALIGN_FUNCTION(); \ |
608 | __irqentry_text_start = .; \ |
609 | *(.irqentry.text) \ |
610 | __irqentry_text_end = .; |
611 | |
612 | #define SOFTIRQENTRY_TEXT \ |
613 | ALIGN_FUNCTION(); \ |
614 | __softirqentry_text_start = .; \ |
615 | *(.softirqentry.text) \ |
616 | __softirqentry_text_end = .; |
617 | |
618 | #define STATIC_CALL_TEXT \ |
619 | ALIGN_FUNCTION(); \ |
620 | __static_call_text_start = .; \ |
621 | *(.static_call.text) \ |
622 | __static_call_text_end = .; |
623 | |
624 | /* Section used for early init (in .S files) */ |
625 | #define HEAD_TEXT KEEP(*(.head.text)) |
626 | |
627 | #define HEAD_TEXT_SECTION \ |
628 | .head.text : AT(ADDR(.head.text) - LOAD_OFFSET) { \ |
629 | HEAD_TEXT \ |
630 | } |
631 | |
632 | /* |
633 | * Exception table |
634 | */ |
635 | #define EXCEPTION_TABLE(align) \ |
636 | . = ALIGN(align); \ |
637 | __ex_table : AT(ADDR(__ex_table) - LOAD_OFFSET) { \ |
638 | BOUNDED_SECTION_BY(__ex_table, ___ex_table) \ |
639 | } |
640 | |
641 | /* |
642 | * .BTF |
643 | */ |
644 | #ifdef CONFIG_DEBUG_INFO_BTF |
645 | #define BTF \ |
646 | .BTF : AT(ADDR(.BTF) - LOAD_OFFSET) { \ |
647 | BOUNDED_SECTION_BY(.BTF, _BTF) \ |
648 | } \ |
649 | . = ALIGN(4); \ |
650 | .BTF_ids : AT(ADDR(.BTF_ids) - LOAD_OFFSET) { \ |
651 | *(.BTF_ids) \ |
652 | } |
653 | #else |
654 | #define BTF |
655 | #endif |
656 | |
657 | /* |
658 | * Init task |
659 | */ |
660 | #define INIT_TASK_DATA_SECTION(align) \ |
661 | . = ALIGN(align); \ |
662 | .data..init_task : AT(ADDR(.data..init_task) - LOAD_OFFSET) { \ |
663 | INIT_TASK_DATA(align) \ |
664 | } |
665 | |
666 | #ifdef CONFIG_CONSTRUCTORS |
667 | #define KERNEL_CTORS() . = ALIGN(8); \ |
668 | __ctors_start = .; \ |
669 | KEEP(*(SORT(.ctors.*))) \ |
670 | KEEP(*(.ctors)) \ |
671 | KEEP(*(SORT(.init_array.*))) \ |
672 | KEEP(*(.init_array)) \ |
673 | __ctors_end = .; |
674 | #else |
675 | #define KERNEL_CTORS() |
676 | #endif |
677 | |
678 | /* init and exit section handling */ |
679 | #define INIT_DATA \ |
680 | KEEP(*(SORT(___kentry+*))) \ |
681 | *(.init.data .init.data.*) \ |
682 | MEM_DISCARD(init.data*) \ |
683 | KERNEL_CTORS() \ |
684 | MCOUNT_REC() \ |
685 | *(.init.rodata .init.rodata.*) \ |
686 | FTRACE_EVENTS() \ |
687 | TRACE_SYSCALLS() \ |
688 | KPROBE_BLACKLIST() \ |
689 | ERROR_INJECT_WHITELIST() \ |
690 | MEM_DISCARD(init.rodata) \ |
691 | CLK_OF_TABLES() \ |
692 | RESERVEDMEM_OF_TABLES() \ |
693 | TIMER_OF_TABLES() \ |
694 | CPU_METHOD_OF_TABLES() \ |
695 | CPUIDLE_METHOD_OF_TABLES() \ |
696 | KERNEL_DTB() \ |
697 | IRQCHIP_OF_MATCH_TABLE() \ |
698 | ACPI_PROBE_TABLE(irqchip) \ |
699 | ACPI_PROBE_TABLE(timer) \ |
700 | THERMAL_TABLE(governor) \ |
701 | EARLYCON_TABLE() \ |
702 | LSM_TABLE() \ |
703 | EARLY_LSM_TABLE() \ |
704 | KUNIT_INIT_TABLE() |
705 | |
706 | #define INIT_TEXT \ |
707 | *(.init.text .init.text.*) \ |
708 | *(.text.startup) \ |
709 | MEM_DISCARD(init.text*) |
710 | |
711 | #define EXIT_DATA \ |
712 | *(.exit.data .exit.data.*) \ |
713 | *(.fini_array .fini_array.*) \ |
714 | *(.dtors .dtors.*) \ |
715 | |
716 | #define EXIT_TEXT \ |
717 | *(.exit.text) \ |
718 | *(.text.exit) \ |
719 | |
720 | #define EXIT_CALL \ |
721 | *(.exitcall.exit) |
722 | |
723 | /* |
724 | * bss (Block Started by Symbol) - uninitialized data |
725 | * zeroed during startup |
726 | */ |
727 | #define SBSS(sbss_align) \ |
728 | . = ALIGN(sbss_align); \ |
729 | .sbss : AT(ADDR(.sbss) - LOAD_OFFSET) { \ |
730 | *(.dynsbss) \ |
731 | *(SBSS_MAIN) \ |
732 | *(.scommon) \ |
733 | } |
734 | |
735 | /* |
736 | * Allow archectures to redefine BSS_FIRST_SECTIONS to add extra |
737 | * sections to the front of bss. |
738 | */ |
739 | #ifndef BSS_FIRST_SECTIONS |
740 | #define BSS_FIRST_SECTIONS |
741 | #endif |
742 | |
743 | #define BSS(bss_align) \ |
744 | . = ALIGN(bss_align); \ |
745 | .bss : AT(ADDR(.bss) - LOAD_OFFSET) { \ |
746 | BSS_FIRST_SECTIONS \ |
747 | . = ALIGN(PAGE_SIZE); \ |
748 | *(.bss..page_aligned) \ |
749 | . = ALIGN(PAGE_SIZE); \ |
750 | *(.dynbss) \ |
751 | *(BSS_MAIN) \ |
752 | *(COMMON) \ |
753 | } |
754 | |
755 | /* |
756 | * DWARF debug sections. |
757 | * Symbols in the DWARF debugging sections are relative to |
758 | * the beginning of the section so we begin them at 0. |
759 | */ |
760 | #define DWARF_DEBUG \ |
761 | /* DWARF 1 */ \ |
762 | .debug 0 : { *(.debug) } \ |
763 | .line 0 : { *(.line) } \ |
764 | /* GNU DWARF 1 extensions */ \ |
765 | .debug_srcinfo 0 : { *(.debug_srcinfo) } \ |
766 | .debug_sfnames 0 : { *(.debug_sfnames) } \ |
767 | /* DWARF 1.1 and DWARF 2 */ \ |
768 | .debug_aranges 0 : { *(.debug_aranges) } \ |
769 | .debug_pubnames 0 : { *(.debug_pubnames) } \ |
770 | /* DWARF 2 */ \ |
771 | .debug_info 0 : { *(.debug_info \ |
772 | .gnu.linkonce.wi.*) } \ |
773 | .debug_abbrev 0 : { *(.debug_abbrev) } \ |
774 | .debug_line 0 : { *(.debug_line) } \ |
775 | .debug_frame 0 : { *(.debug_frame) } \ |
776 | .debug_str 0 : { *(.debug_str) } \ |
777 | .debug_loc 0 : { *(.debug_loc) } \ |
778 | .debug_macinfo 0 : { *(.debug_macinfo) } \ |
779 | .debug_pubtypes 0 : { *(.debug_pubtypes) } \ |
780 | /* DWARF 3 */ \ |
781 | .debug_ranges 0 : { *(.debug_ranges) } \ |
782 | /* SGI/MIPS DWARF 2 extensions */ \ |
783 | .debug_weaknames 0 : { *(.debug_weaknames) } \ |
784 | .debug_funcnames 0 : { *(.debug_funcnames) } \ |
785 | .debug_typenames 0 : { *(.debug_typenames) } \ |
786 | .debug_varnames 0 : { *(.debug_varnames) } \ |
787 | /* GNU DWARF 2 extensions */ \ |
788 | .debug_gnu_pubnames 0 : { *(.debug_gnu_pubnames) } \ |
789 | .debug_gnu_pubtypes 0 : { *(.debug_gnu_pubtypes) } \ |
790 | /* DWARF 4 */ \ |
791 | .debug_types 0 : { *(.debug_types) } \ |
792 | /* DWARF 5 */ \ |
793 | .debug_addr 0 : { *(.debug_addr) } \ |
794 | .debug_line_str 0 : { *(.debug_line_str) } \ |
795 | .debug_loclists 0 : { *(.debug_loclists) } \ |
796 | .debug_macro 0 : { *(.debug_macro) } \ |
797 | .debug_names 0 : { *(.debug_names) } \ |
798 | .debug_rnglists 0 : { *(.debug_rnglists) } \ |
799 | .debug_str_offsets 0 : { *(.debug_str_offsets) } |
800 | |
801 | /* Stabs debugging sections. */ |
802 | #define STABS_DEBUG \ |
803 | .stab 0 : { *(.stab) } \ |
804 | .stabstr 0 : { *(.stabstr) } \ |
805 | .stab.excl 0 : { *(.stab.excl) } \ |
806 | .stab.exclstr 0 : { *(.stab.exclstr) } \ |
807 | .stab.index 0 : { *(.stab.index) } \ |
808 | .stab.indexstr 0 : { *(.stab.indexstr) } |
809 | |
810 | /* Required sections not related to debugging. */ |
811 | #define ELF_DETAILS \ |
812 | .comment 0 : { *(.comment) } \ |
813 | .symtab 0 : { *(.symtab) } \ |
814 | .strtab 0 : { *(.strtab) } \ |
815 | .shstrtab 0 : { *(.shstrtab) } |
816 | |
817 | #ifdef CONFIG_GENERIC_BUG |
818 | #define BUG_TABLE \ |
819 | . = ALIGN(8); \ |
820 | __bug_table : AT(ADDR(__bug_table) - LOAD_OFFSET) { \ |
821 | BOUNDED_SECTION_BY(__bug_table, ___bug_table) \ |
822 | } |
823 | #else |
824 | #define BUG_TABLE |
825 | #endif |
826 | |
827 | #ifdef CONFIG_UNWINDER_ORC |
828 | #define ORC_UNWIND_TABLE \ |
829 | .orc_header : AT(ADDR(.orc_header) - LOAD_OFFSET) { \ |
830 | BOUNDED_SECTION_BY(.orc_header, _orc_header) \ |
831 | } \ |
832 | . = ALIGN(4); \ |
833 | .orc_unwind_ip : AT(ADDR(.orc_unwind_ip) - LOAD_OFFSET) { \ |
834 | BOUNDED_SECTION_BY(.orc_unwind_ip, _orc_unwind_ip) \ |
835 | } \ |
836 | . = ALIGN(2); \ |
837 | .orc_unwind : AT(ADDR(.orc_unwind) - LOAD_OFFSET) { \ |
838 | BOUNDED_SECTION_BY(.orc_unwind, _orc_unwind) \ |
839 | } \ |
840 | text_size = _etext - _stext; \ |
841 | . = ALIGN(4); \ |
842 | .orc_lookup : AT(ADDR(.orc_lookup) - LOAD_OFFSET) { \ |
843 | orc_lookup = .; \ |
844 | . += (((text_size + LOOKUP_BLOCK_SIZE - 1) / \ |
845 | LOOKUP_BLOCK_SIZE) + 1) * 4; \ |
846 | orc_lookup_end = .; \ |
847 | } |
848 | #else |
849 | #define ORC_UNWIND_TABLE |
850 | #endif |
851 | |
852 | /* Built-in firmware blobs */ |
853 | #ifdef CONFIG_FW_LOADER |
854 | #define FW_LOADER_BUILT_IN_DATA \ |
855 | .builtin_fw : AT(ADDR(.builtin_fw) - LOAD_OFFSET) ALIGN(8) { \ |
856 | BOUNDED_SECTION_PRE_LABEL(.builtin_fw, _builtin_fw, __start, __end) \ |
857 | } |
858 | #else |
859 | #define FW_LOADER_BUILT_IN_DATA |
860 | #endif |
861 | |
862 | #ifdef CONFIG_PM_TRACE |
863 | #define TRACEDATA \ |
864 | . = ALIGN(4); \ |
865 | .tracedata : AT(ADDR(.tracedata) - LOAD_OFFSET) { \ |
866 | BOUNDED_SECTION_POST_LABEL(.tracedata, __tracedata, _start, _end) \ |
867 | } |
868 | #else |
869 | #define TRACEDATA |
870 | #endif |
871 | |
872 | #ifdef CONFIG_PRINTK_INDEX |
873 | #define PRINTK_INDEX \ |
874 | .printk_index : AT(ADDR(.printk_index) - LOAD_OFFSET) { \ |
875 | BOUNDED_SECTION_BY(.printk_index, _printk_index) \ |
876 | } |
877 | #else |
878 | #define PRINTK_INDEX |
879 | #endif |
880 | |
881 | /* |
882 | * Discard .note.GNU-stack, which is emitted as PROGBITS by the compiler. |
883 | * Otherwise, the type of .notes section would become PROGBITS instead of NOTES. |
884 | * |
885 | * Also, discard .note.gnu.property, otherwise it forces the notes section to |
886 | * be 8-byte aligned which causes alignment mismatches with the kernel's custom |
887 | * 4-byte aligned notes. |
888 | */ |
889 | #define NOTES \ |
890 | /DISCARD/ : { \ |
891 | *(.note.GNU-stack) \ |
892 | *(.note.gnu.property) \ |
893 | } \ |
894 | .notes : AT(ADDR(.notes) - LOAD_OFFSET) { \ |
895 | BOUNDED_SECTION_BY(.note.*, _notes) \ |
896 | } NOTES_HEADERS \ |
897 | NOTES_HEADERS_RESTORE |
898 | |
899 | #define INIT_SETUP(initsetup_align) \ |
900 | . = ALIGN(initsetup_align); \ |
901 | BOUNDED_SECTION_POST_LABEL(.init.setup, __setup, _start, _end) |
902 | |
903 | #define INIT_CALLS_LEVEL(level) \ |
904 | __initcall##level##_start = .; \ |
905 | KEEP(*(.initcall##level##.init)) \ |
906 | KEEP(*(.initcall##level##s.init)) \ |
907 | |
908 | #define INIT_CALLS \ |
909 | __initcall_start = .; \ |
910 | KEEP(*(.initcallearly.init)) \ |
911 | INIT_CALLS_LEVEL(0) \ |
912 | INIT_CALLS_LEVEL(1) \ |
913 | INIT_CALLS_LEVEL(2) \ |
914 | INIT_CALLS_LEVEL(3) \ |
915 | INIT_CALLS_LEVEL(4) \ |
916 | INIT_CALLS_LEVEL(5) \ |
917 | INIT_CALLS_LEVEL(rootfs) \ |
918 | INIT_CALLS_LEVEL(6) \ |
919 | INIT_CALLS_LEVEL(7) \ |
920 | __initcall_end = .; |
921 | |
922 | #define CON_INITCALL \ |
923 | BOUNDED_SECTION_POST_LABEL(.con_initcall.init, __con_initcall, _start, _end) |
924 | |
925 | /* Alignment must be consistent with (kunit_suite *) in include/kunit/test.h */ |
926 | #define KUNIT_TABLE() \ |
927 | . = ALIGN(8); \ |
928 | BOUNDED_SECTION_POST_LABEL(.kunit_test_suites, __kunit_suites, _start, _end) |
929 | |
930 | /* Alignment must be consistent with (kunit_suite *) in include/kunit/test.h */ |
931 | #define KUNIT_INIT_TABLE() \ |
932 | . = ALIGN(8); \ |
933 | BOUNDED_SECTION_POST_LABEL(.kunit_init_test_suites, \ |
934 | __kunit_init_suites, _start, _end) |
935 | |
936 | #ifdef CONFIG_BLK_DEV_INITRD |
937 | #define INIT_RAM_FS \ |
938 | . = ALIGN(4); \ |
939 | __initramfs_start = .; \ |
940 | KEEP(*(.init.ramfs)) \ |
941 | . = ALIGN(8); \ |
942 | KEEP(*(.init.ramfs.info)) |
943 | #else |
944 | #define INIT_RAM_FS |
945 | #endif |
946 | |
947 | /* |
948 | * Memory encryption operates on a page basis. Since we need to clear |
949 | * the memory encryption mask for this section, it needs to be aligned |
950 | * on a page boundary and be a page-size multiple in length. |
951 | * |
952 | * Note: We use a separate section so that only this section gets |
953 | * decrypted to avoid exposing more than we wish. |
954 | */ |
955 | #ifdef CONFIG_AMD_MEM_ENCRYPT |
956 | #define PERCPU_DECRYPTED_SECTION \ |
957 | . = ALIGN(PAGE_SIZE); \ |
958 | *(.data..percpu..decrypted) \ |
959 | . = ALIGN(PAGE_SIZE); |
960 | #else |
961 | #define PERCPU_DECRYPTED_SECTION |
962 | #endif |
963 | |
964 | |
965 | /* |
966 | * Default discarded sections. |
967 | * |
968 | * Some archs want to discard exit text/data at runtime rather than |
969 | * link time due to cross-section references such as alt instructions, |
970 | * bug table, eh_frame, etc. DISCARDS must be the last of output |
971 | * section definitions so that such archs put those in earlier section |
972 | * definitions. |
973 | */ |
974 | #ifdef RUNTIME_DISCARD_EXIT |
975 | #define EXIT_DISCARDS |
976 | #else |
977 | #define EXIT_DISCARDS \ |
978 | EXIT_TEXT \ |
979 | EXIT_DATA |
980 | #endif |
981 | |
982 | /* |
983 | * Clang's -fprofile-arcs, -fsanitize=kernel-address, and |
984 | * -fsanitize=thread produce unwanted sections (.eh_frame |
985 | * and .init_array.*), but CONFIG_CONSTRUCTORS wants to |
986 | * keep any .init_array.* sections. |
987 | * https://llvm.org/pr46478 |
988 | */ |
989 | #ifdef CONFIG_UNWIND_TABLES |
990 | #define DISCARD_EH_FRAME |
991 | #else |
992 | #define DISCARD_EH_FRAME *(.eh_frame) |
993 | #endif |
994 | #if defined(CONFIG_GCOV_KERNEL) || defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KCSAN) |
995 | # ifdef CONFIG_CONSTRUCTORS |
996 | # define SANITIZER_DISCARDS \ |
997 | DISCARD_EH_FRAME |
998 | # else |
999 | # define SANITIZER_DISCARDS \ |
1000 | *(.init_array) *(.init_array.*) \ |
1001 | DISCARD_EH_FRAME |
1002 | # endif |
1003 | #else |
1004 | # define SANITIZER_DISCARDS |
1005 | #endif |
1006 | |
1007 | #define COMMON_DISCARDS \ |
1008 | SANITIZER_DISCARDS \ |
1009 | PATCHABLE_DISCARDS \ |
1010 | *(.discard) \ |
1011 | *(.discard.*) \ |
1012 | *(.export_symbol) \ |
1013 | *(.modinfo) \ |
1014 | /* ld.bfd warns about .gnu.version* even when not emitted */ \ |
1015 | *(.gnu.version*) \ |
1016 | |
1017 | #define DISCARDS \ |
1018 | /DISCARD/ : { \ |
1019 | EXIT_DISCARDS \ |
1020 | EXIT_CALL \ |
1021 | COMMON_DISCARDS \ |
1022 | } |
1023 | |
1024 | /** |
1025 | * PERCPU_INPUT - the percpu input sections |
1026 | * @cacheline: cacheline size |
1027 | * |
1028 | * The core percpu section names and core symbols which do not rely |
1029 | * directly upon load addresses. |
1030 | * |
1031 | * @cacheline is used to align subsections to avoid false cacheline |
1032 | * sharing between subsections for different purposes. |
1033 | */ |
1034 | #define PERCPU_INPUT(cacheline) \ |
1035 | __per_cpu_start = .; \ |
1036 | *(.data..percpu..first) \ |
1037 | . = ALIGN(PAGE_SIZE); \ |
1038 | *(.data..percpu..page_aligned) \ |
1039 | . = ALIGN(cacheline); \ |
1040 | *(.data..percpu..read_mostly) \ |
1041 | . = ALIGN(cacheline); \ |
1042 | *(.data..percpu) \ |
1043 | *(.data..percpu..shared_aligned) \ |
1044 | PERCPU_DECRYPTED_SECTION \ |
1045 | __per_cpu_end = .; |
1046 | |
1047 | /** |
1048 | * PERCPU_VADDR - define output section for percpu area |
1049 | * @cacheline: cacheline size |
1050 | * @vaddr: explicit base address (optional) |
1051 | * @phdr: destination PHDR (optional) |
1052 | * |
1053 | * Macro which expands to output section for percpu area. |
1054 | * |
1055 | * @cacheline is used to align subsections to avoid false cacheline |
1056 | * sharing between subsections for different purposes. |
1057 | * |
1058 | * If @vaddr is not blank, it specifies explicit base address and all |
1059 | * percpu symbols will be offset from the given address. If blank, |
1060 | * @vaddr always equals @laddr + LOAD_OFFSET. |
1061 | * |
1062 | * @phdr defines the output PHDR to use if not blank. Be warned that |
1063 | * output PHDR is sticky. If @phdr is specified, the next output |
1064 | * section in the linker script will go there too. @phdr should have |
1065 | * a leading colon. |
1066 | * |
1067 | * Note that this macros defines __per_cpu_load as an absolute symbol. |
1068 | * If there is no need to put the percpu section at a predetermined |
1069 | * address, use PERCPU_SECTION. |
1070 | */ |
1071 | #define PERCPU_VADDR(cacheline, vaddr, phdr) \ |
1072 | __per_cpu_load = .; \ |
1073 | .data..percpu vaddr : AT(__per_cpu_load - LOAD_OFFSET) { \ |
1074 | PERCPU_INPUT(cacheline) \ |
1075 | } phdr \ |
1076 | . = __per_cpu_load + SIZEOF(.data..percpu); |
1077 | |
1078 | /** |
1079 | * PERCPU_SECTION - define output section for percpu area, simple version |
1080 | * @cacheline: cacheline size |
1081 | * |
1082 | * Align to PAGE_SIZE and outputs output section for percpu area. This |
1083 | * macro doesn't manipulate @vaddr or @phdr and __per_cpu_load and |
1084 | * __per_cpu_start will be identical. |
1085 | * |
1086 | * This macro is equivalent to ALIGN(PAGE_SIZE); PERCPU_VADDR(@cacheline,,) |
1087 | * except that __per_cpu_load is defined as a relative symbol against |
1088 | * .data..percpu which is required for relocatable x86_32 configuration. |
1089 | */ |
1090 | #define PERCPU_SECTION(cacheline) \ |
1091 | . = ALIGN(PAGE_SIZE); \ |
1092 | .data..percpu : AT(ADDR(.data..percpu) - LOAD_OFFSET) { \ |
1093 | __per_cpu_load = .; \ |
1094 | PERCPU_INPUT(cacheline) \ |
1095 | } |
1096 | |
1097 | |
1098 | /* |
1099 | * Definition of the high level *_SECTION macros |
1100 | * They will fit only a subset of the architectures |
1101 | */ |
1102 | |
1103 | |
1104 | /* |
1105 | * Writeable data. |
1106 | * All sections are combined in a single .data section. |
1107 | * The sections following CONSTRUCTORS are arranged so their |
1108 | * typical alignment matches. |
1109 | * A cacheline is typical/always less than a PAGE_SIZE so |
1110 | * the sections that has this restriction (or similar) |
1111 | * is located before the ones requiring PAGE_SIZE alignment. |
1112 | * NOSAVE_DATA starts and ends with a PAGE_SIZE alignment which |
1113 | * matches the requirement of PAGE_ALIGNED_DATA. |
1114 | * |
1115 | * use 0 as page_align if page_aligned data is not used */ |
1116 | #define RW_DATA(cacheline, pagealigned, inittask) \ |
1117 | . = ALIGN(PAGE_SIZE); \ |
1118 | .data : AT(ADDR(.data) - LOAD_OFFSET) { \ |
1119 | INIT_TASK_DATA(inittask) \ |
1120 | NOSAVE_DATA \ |
1121 | PAGE_ALIGNED_DATA(pagealigned) \ |
1122 | CACHELINE_ALIGNED_DATA(cacheline) \ |
1123 | READ_MOSTLY_DATA(cacheline) \ |
1124 | DATA_DATA \ |
1125 | CONSTRUCTORS \ |
1126 | } \ |
1127 | BUG_TABLE \ |
1128 | |
1129 | #define INIT_TEXT_SECTION(inittext_align) \ |
1130 | . = ALIGN(inittext_align); \ |
1131 | .init.text : AT(ADDR(.init.text) - LOAD_OFFSET) { \ |
1132 | _sinittext = .; \ |
1133 | INIT_TEXT \ |
1134 | _einittext = .; \ |
1135 | } |
1136 | |
1137 | #define INIT_DATA_SECTION(initsetup_align) \ |
1138 | .init.data : AT(ADDR(.init.data) - LOAD_OFFSET) { \ |
1139 | INIT_DATA \ |
1140 | INIT_SETUP(initsetup_align) \ |
1141 | INIT_CALLS \ |
1142 | CON_INITCALL \ |
1143 | INIT_RAM_FS \ |
1144 | } |
1145 | |
1146 | #define BSS_SECTION(sbss_align, bss_align, stop_align) \ |
1147 | . = ALIGN(sbss_align); \ |
1148 | __bss_start = .; \ |
1149 | SBSS(sbss_align) \ |
1150 | BSS(bss_align) \ |
1151 | . = ALIGN(stop_align); \ |
1152 | __bss_stop = .; |
1153 | |