1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Adapted from arm64 version. |
4 | * |
5 | * Copyright (C) 2012 ARM Limited |
6 | * Copyright (C) 2015 Mentor Graphics Corporation. |
7 | */ |
8 | |
9 | #include <linux/cache.h> |
10 | #include <linux/elf.h> |
11 | #include <linux/err.h> |
12 | #include <linux/kernel.h> |
13 | #include <linux/mm.h> |
14 | #include <linux/of.h> |
15 | #include <linux/printk.h> |
16 | #include <linux/slab.h> |
17 | #include <linux/timekeeper_internal.h> |
18 | #include <linux/vmalloc.h> |
19 | #include <asm/arch_timer.h> |
20 | #include <asm/barrier.h> |
21 | #include <asm/cacheflush.h> |
22 | #include <asm/page.h> |
23 | #include <asm/vdso.h> |
24 | #include <clocksource/arm_arch_timer.h> |
25 | #include <vdso/helpers.h> |
26 | #include <vdso/vsyscall.h> |
27 | |
28 | #define MAX_SYMNAME 64 |
29 | |
30 | static struct page **vdso_text_pagelist; |
31 | |
32 | extern char vdso_start[], vdso_end[]; |
33 | |
34 | /* Total number of pages needed for the data and text portions of the VDSO. */ |
35 | unsigned int vdso_total_pages __ro_after_init; |
36 | |
37 | static union vdso_data_store vdso_data_store __page_aligned_data; |
38 | struct vdso_data *vdso_data = vdso_data_store.data; |
39 | |
40 | static struct page *vdso_data_page __ro_after_init; |
41 | static const struct vm_special_mapping vdso_data_mapping = { |
42 | .name = "[vvar]" , |
43 | .pages = &vdso_data_page, |
44 | }; |
45 | |
46 | static int vdso_mremap(const struct vm_special_mapping *sm, |
47 | struct vm_area_struct *new_vma) |
48 | { |
49 | current->mm->context.vdso = new_vma->vm_start; |
50 | |
51 | return 0; |
52 | } |
53 | |
54 | static struct vm_special_mapping vdso_text_mapping __ro_after_init = { |
55 | .name = "[vdso]" , |
56 | .mremap = vdso_mremap, |
57 | }; |
58 | |
59 | struct elfinfo { |
60 | Elf32_Ehdr *hdr; /* ptr to ELF */ |
61 | Elf32_Sym *dynsym; /* ptr to .dynsym section */ |
62 | unsigned long dynsymsize; /* size of .dynsym section */ |
63 | char *dynstr; /* ptr to .dynstr section */ |
64 | }; |
65 | |
66 | /* Cached result of boot-time check for whether the arch timer exists, |
67 | * and if so, whether the virtual counter is useable. |
68 | */ |
69 | bool cntvct_ok __ro_after_init; |
70 | |
71 | static bool __init cntvct_functional(void) |
72 | { |
73 | struct device_node *np; |
74 | bool ret = false; |
75 | |
76 | if (!IS_ENABLED(CONFIG_ARM_ARCH_TIMER)) |
77 | goto out; |
78 | |
79 | /* The arm_arch_timer core should export |
80 | * arch_timer_use_virtual or similar so we don't have to do |
81 | * this. |
82 | */ |
83 | np = of_find_compatible_node(NULL, NULL, compat: "arm,armv7-timer" ); |
84 | if (!np) |
85 | np = of_find_compatible_node(NULL, NULL, compat: "arm,armv8-timer" ); |
86 | if (!np) |
87 | goto out_put; |
88 | |
89 | if (of_property_read_bool(np, propname: "arm,cpu-registers-not-fw-configured" )) |
90 | goto out_put; |
91 | |
92 | ret = true; |
93 | |
94 | out_put: |
95 | of_node_put(node: np); |
96 | out: |
97 | return ret; |
98 | } |
99 | |
100 | static void * __init find_section(Elf32_Ehdr *ehdr, const char *name, |
101 | unsigned long *size) |
102 | { |
103 | Elf32_Shdr *sechdrs; |
104 | unsigned int i; |
105 | char *secnames; |
106 | |
107 | /* Grab section headers and strings so we can tell who is who */ |
108 | sechdrs = (void *)ehdr + ehdr->e_shoff; |
109 | secnames = (void *)ehdr + sechdrs[ehdr->e_shstrndx].sh_offset; |
110 | |
111 | /* Find the section they want */ |
112 | for (i = 1; i < ehdr->e_shnum; i++) { |
113 | if (strcmp(secnames + sechdrs[i].sh_name, name) == 0) { |
114 | if (size) |
115 | *size = sechdrs[i].sh_size; |
116 | return (void *)ehdr + sechdrs[i].sh_offset; |
117 | } |
118 | } |
119 | |
120 | if (size) |
121 | *size = 0; |
122 | return NULL; |
123 | } |
124 | |
125 | static Elf32_Sym * __init find_symbol(struct elfinfo *lib, const char *symname) |
126 | { |
127 | unsigned int i; |
128 | |
129 | for (i = 0; i < (lib->dynsymsize / sizeof(Elf32_Sym)); i++) { |
130 | char name[MAX_SYMNAME], *c; |
131 | |
132 | if (lib->dynsym[i].st_name == 0) |
133 | continue; |
134 | strscpy(name, lib->dynstr + lib->dynsym[i].st_name, |
135 | MAX_SYMNAME); |
136 | c = strchr(name, '@'); |
137 | if (c) |
138 | *c = 0; |
139 | if (strcmp(symname, name) == 0) |
140 | return &lib->dynsym[i]; |
141 | } |
142 | return NULL; |
143 | } |
144 | |
145 | static void __init vdso_nullpatch_one(struct elfinfo *lib, const char *symname) |
146 | { |
147 | Elf32_Sym *sym; |
148 | |
149 | sym = find_symbol(lib, symname); |
150 | if (!sym) |
151 | return; |
152 | |
153 | sym->st_name = 0; |
154 | } |
155 | |
156 | static void __init patch_vdso(void *ehdr) |
157 | { |
158 | struct elfinfo einfo; |
159 | |
160 | einfo = (struct elfinfo) { |
161 | .hdr = ehdr, |
162 | }; |
163 | |
164 | einfo.dynsym = find_section(ehdr: einfo.hdr, name: ".dynsym" , size: &einfo.dynsymsize); |
165 | einfo.dynstr = find_section(ehdr: einfo.hdr, name: ".dynstr" , NULL); |
166 | |
167 | /* If the virtual counter is absent or non-functional we don't |
168 | * want programs to incur the slight additional overhead of |
169 | * dispatching through the VDSO only to fall back to syscalls. |
170 | */ |
171 | if (!cntvct_ok) { |
172 | vdso_nullpatch_one(lib: &einfo, symname: "__vdso_gettimeofday" ); |
173 | vdso_nullpatch_one(lib: &einfo, symname: "__vdso_clock_gettime" ); |
174 | vdso_nullpatch_one(lib: &einfo, symname: "__vdso_clock_gettime64" ); |
175 | } |
176 | } |
177 | |
178 | static int __init vdso_init(void) |
179 | { |
180 | unsigned int text_pages; |
181 | int i; |
182 | |
183 | if (memcmp(p: vdso_start, q: "\177ELF" , size: 4)) { |
184 | pr_err("VDSO is not a valid ELF object!\n" ); |
185 | return -ENOEXEC; |
186 | } |
187 | |
188 | text_pages = (vdso_end - vdso_start) >> PAGE_SHIFT; |
189 | |
190 | /* Allocate the VDSO text pagelist */ |
191 | vdso_text_pagelist = kcalloc(n: text_pages, size: sizeof(struct page *), |
192 | GFP_KERNEL); |
193 | if (vdso_text_pagelist == NULL) |
194 | return -ENOMEM; |
195 | |
196 | /* Grab the VDSO data page. */ |
197 | vdso_data_page = virt_to_page(vdso_data); |
198 | |
199 | /* Grab the VDSO text pages. */ |
200 | for (i = 0; i < text_pages; i++) { |
201 | struct page *page; |
202 | |
203 | page = virt_to_page(vdso_start + i * PAGE_SIZE); |
204 | vdso_text_pagelist[i] = page; |
205 | } |
206 | |
207 | vdso_text_mapping.pages = vdso_text_pagelist; |
208 | |
209 | vdso_total_pages = 1; /* for the data/vvar page */ |
210 | vdso_total_pages += text_pages; |
211 | |
212 | cntvct_ok = cntvct_functional(); |
213 | |
214 | patch_vdso(ehdr: vdso_start); |
215 | |
216 | return 0; |
217 | } |
218 | arch_initcall(vdso_init); |
219 | |
220 | static int install_vvar(struct mm_struct *mm, unsigned long addr) |
221 | { |
222 | struct vm_area_struct *vma; |
223 | |
224 | vma = _install_special_mapping(mm, addr, PAGE_SIZE, |
225 | VM_READ | VM_MAYREAD, |
226 | spec: &vdso_data_mapping); |
227 | |
228 | return PTR_ERR_OR_ZERO(ptr: vma); |
229 | } |
230 | |
231 | /* assumes mmap_lock is write-locked */ |
232 | void arm_install_vdso(struct mm_struct *mm, unsigned long addr) |
233 | { |
234 | struct vm_area_struct *vma; |
235 | unsigned long len; |
236 | |
237 | mm->context.vdso = 0; |
238 | |
239 | if (vdso_text_pagelist == NULL) |
240 | return; |
241 | |
242 | if (install_vvar(mm, addr)) |
243 | return; |
244 | |
245 | /* Account for vvar page. */ |
246 | addr += PAGE_SIZE; |
247 | len = (vdso_total_pages - 1) << PAGE_SHIFT; |
248 | |
249 | vma = _install_special_mapping(mm, addr, len, |
250 | VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC, |
251 | spec: &vdso_text_mapping); |
252 | |
253 | if (!IS_ERR(ptr: vma)) |
254 | mm->context.vdso = addr; |
255 | } |
256 | |
257 | |