1 | /* Manage function descriptors. Generic version. |
2 | Copyright (C) 1999-2022 Free Software Foundation, Inc. |
3 | This file is part of the GNU C Library. |
4 | |
5 | The GNU C Library is free software; you can redistribute it and/or |
6 | modify it under the terms of the GNU Lesser General Public |
7 | License as published by the Free Software Foundation; either |
8 | version 2.1 of the License, or (at your option) any later version. |
9 | |
10 | The GNU C Library is distributed in the hope that it will be useful, |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
13 | Lesser General Public License for more details. |
14 | |
15 | You should have received a copy of the GNU Lesser General Public |
16 | License along with the GNU C Library; if not, write to the Free |
17 | Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA |
18 | 02111-1307 USA. */ |
19 | |
20 | #include <libintl.h> |
21 | #include <unistd.h> |
22 | #include <string.h> |
23 | #include <sys/param.h> |
24 | #include <sys/mman.h> |
25 | #include <link.h> |
26 | #include <ldsodefs.h> |
27 | #include <elf/dynamic-link.h> |
28 | #include <dl-fptr.h> |
29 | #include <dl-runtime.h> |
30 | #include <dl-unmap-segments.h> |
31 | #include <atomic.h> |
32 | #include <libc-pointer-arith.h> |
33 | |
34 | #ifndef ELF_MACHINE_BOOT_FPTR_TABLE_LEN |
35 | /* ELF_MACHINE_BOOT_FPTR_TABLE_LEN should be greater than the number of |
36 | dynamic symbols in ld.so. */ |
37 | # define ELF_MACHINE_BOOT_FPTR_TABLE_LEN 256 |
38 | #endif |
39 | |
40 | #ifndef ELF_MACHINE_LOAD_ADDRESS |
41 | # error "ELF_MACHINE_LOAD_ADDRESS is not defined." |
42 | #endif |
43 | |
44 | #ifndef COMPARE_AND_SWAP |
45 | # define COMPARE_AND_SWAP(ptr, old, new) \ |
46 | (catomic_compare_and_exchange_bool_acq (ptr, new, old) == 0) |
47 | #endif |
48 | |
49 | ElfW(Addr) _dl_boot_fptr_table [ELF_MACHINE_BOOT_FPTR_TABLE_LEN]; |
50 | |
51 | static struct local |
52 | { |
53 | struct fdesc_table *root; |
54 | struct fdesc *free_list; |
55 | unsigned int npages; /* # of pages to allocate */ |
56 | /* the next to members MUST be consecutive! */ |
57 | struct fdesc_table boot_table; |
58 | struct fdesc boot_fdescs[1024]; |
59 | } |
60 | local = |
61 | { |
62 | #ifdef SHARED |
63 | /* Address of .boot_table is not known until runtime. */ |
64 | .root = 0, |
65 | #else |
66 | .root = &local.boot_table, |
67 | #endif |
68 | .npages = 2, |
69 | .boot_table = |
70 | { |
71 | .len = sizeof (local.boot_fdescs) / sizeof (local.boot_fdescs[0]), |
72 | .first_unused = 0 |
73 | } |
74 | }; |
75 | |
76 | /* Create a new fdesc table and return a pointer to the first fdesc |
77 | entry. The fdesc lock must have been acquired already. */ |
78 | |
79 | static struct fdesc_table * |
80 | new_fdesc_table (struct local *l, size_t *size) |
81 | { |
82 | size_t old_npages = l->npages; |
83 | size_t new_npages = old_npages + old_npages; |
84 | struct fdesc_table *new_table; |
85 | |
86 | /* If someone has just created a new table, we return NULL to tell |
87 | the caller to use the new table. */ |
88 | if (! COMPARE_AND_SWAP (&l->npages, old_npages, new_npages)) |
89 | return (struct fdesc_table *) NULL; |
90 | |
91 | *size = old_npages * GLRO(dl_pagesize); |
92 | new_table = __mmap (NULL, *size, |
93 | PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0); |
94 | if (new_table == MAP_FAILED) |
95 | _dl_signal_error (errno, NULL, NULL, |
96 | N_("cannot map pages for fdesc table" )); |
97 | |
98 | new_table->len |
99 | = (*size - sizeof (*new_table)) / sizeof (struct fdesc); |
100 | new_table->first_unused = 1; |
101 | return new_table; |
102 | } |
103 | |
104 | /* Must call _dl_fptr_init before using any other function. */ |
105 | void |
106 | _dl_fptr_init (void) |
107 | { |
108 | struct local *l; |
109 | |
110 | ELF_MACHINE_LOAD_ADDRESS (l, local); |
111 | l->root = &l->boot_table; |
112 | } |
113 | |
114 | static ElfW(Addr) |
115 | make_fdesc (ElfW(Addr) ip, ElfW(Addr) gp) |
116 | { |
117 | struct fdesc *fdesc = NULL; |
118 | struct fdesc_table *root; |
119 | unsigned int old; |
120 | struct local *l; |
121 | |
122 | ELF_MACHINE_LOAD_ADDRESS (l, local); |
123 | |
124 | retry: |
125 | root = l->root; |
126 | while (1) |
127 | { |
128 | old = root->first_unused; |
129 | if (old >= root->len) |
130 | break; |
131 | else if (COMPARE_AND_SWAP (&root->first_unused, old, old + 1)) |
132 | { |
133 | fdesc = &root->fdesc[old]; |
134 | goto install; |
135 | } |
136 | } |
137 | |
138 | if (l->free_list) |
139 | { |
140 | /* Get it from free-list. */ |
141 | do |
142 | { |
143 | fdesc = l->free_list; |
144 | if (fdesc == NULL) |
145 | goto retry; |
146 | } |
147 | while (! COMPARE_AND_SWAP ((ElfW(Addr) *) &l->free_list, |
148 | (ElfW(Addr)) fdesc, fdesc->ip)); |
149 | } |
150 | else |
151 | { |
152 | /* Create a new fdesc table. */ |
153 | size_t size; |
154 | struct fdesc_table *new_table = new_fdesc_table (l, size: &size); |
155 | |
156 | if (new_table == NULL) |
157 | goto retry; |
158 | |
159 | new_table->next = root; |
160 | if (! COMPARE_AND_SWAP ((ElfW(Addr) *) &l->root, |
161 | (ElfW(Addr)) root, |
162 | (ElfW(Addr)) new_table)) |
163 | { |
164 | /* Someone has just installed a new table. Return NULL to |
165 | tell the caller to use the new table. */ |
166 | __munmap (new_table, size); |
167 | goto retry; |
168 | } |
169 | |
170 | /* Note that the first entry was reserved while allocating the |
171 | memory for the new page. */ |
172 | fdesc = &new_table->fdesc[0]; |
173 | } |
174 | |
175 | install: |
176 | fdesc->gp = gp; |
177 | fdesc->ip = ip; |
178 | |
179 | return (ElfW(Addr)) fdesc; |
180 | } |
181 | |
182 | |
183 | static inline ElfW(Addr) * __attribute__ ((always_inline)) |
184 | make_fptr_table (struct link_map *map) |
185 | { |
186 | const ElfW(Sym) *symtab = (const void *) D_PTR (map, l_info[DT_SYMTAB]); |
187 | const char *strtab = (const void *) D_PTR (map, l_info[DT_STRTAB]); |
188 | ElfW(Addr) *fptr_table; |
189 | size_t size; |
190 | size_t len; |
191 | const ElfW(Sym) *symtabend; |
192 | |
193 | /* Determine the end of the dynamic symbol table using the hash. */ |
194 | if (map->l_info[DT_HASH] != NULL) |
195 | symtabend = (symtab + ((Elf_Symndx *) D_PTR (map, l_info[DT_HASH]))[1]); |
196 | else |
197 | /* There is no direct way to determine the number of symbols in the |
198 | dynamic symbol table and no hash table is present. The ELF |
199 | binary is ill-formed but what shall we do? Use the beginning of |
200 | the string table which generally follows the symbol table. */ |
201 | symtabend = (const ElfW(Sym) *) strtab; |
202 | |
203 | len = (((char *) symtabend - (char *) symtab) |
204 | / map->l_info[DT_SYMENT]->d_un.d_val); |
205 | size = ALIGN_UP (len * sizeof (fptr_table[0]), GLRO(dl_pagesize)); |
206 | |
207 | /* We don't support systems without MAP_ANON. We avoid using malloc |
208 | because this might get called before malloc is setup. */ |
209 | fptr_table = __mmap (NULL, size, |
210 | PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, |
211 | -1, 0); |
212 | if (fptr_table == MAP_FAILED) |
213 | _dl_signal_error (errno, NULL, NULL, |
214 | N_("cannot map pages for fptr table" )); |
215 | |
216 | if (COMPARE_AND_SWAP ((ElfW(Addr) *) &map->l_mach.fptr_table, |
217 | (ElfW(Addr)) NULL, (ElfW(Addr)) fptr_table)) |
218 | map->l_mach.fptr_table_len = len; |
219 | else |
220 | __munmap (fptr_table, len * sizeof (fptr_table[0])); |
221 | |
222 | return map->l_mach.fptr_table; |
223 | } |
224 | |
225 | |
226 | ElfW(Addr) |
227 | _dl_make_fptr (struct link_map *map, const ElfW(Sym) *sym, |
228 | ElfW(Addr) ip) |
229 | { |
230 | ElfW(Addr) *ftab = map->l_mach.fptr_table; |
231 | const ElfW(Sym) *symtab; |
232 | Elf_Symndx symidx; |
233 | struct local *l; |
234 | |
235 | if (__builtin_expect (ftab == NULL, 0)) |
236 | ftab = make_fptr_table (map); |
237 | |
238 | symtab = (const void *) D_PTR (map, l_info[DT_SYMTAB]); |
239 | symidx = sym - symtab; |
240 | |
241 | if (symidx >= map->l_mach.fptr_table_len) |
242 | _dl_signal_error (0, NULL, NULL, |
243 | N_("internal error: symidx out of range of fptr table" )); |
244 | |
245 | while (ftab[symidx] == 0) |
246 | { |
247 | /* GOT has already been relocated in elf_get_dynamic_info - |
248 | don't try to relocate it again. */ |
249 | ElfW(Addr) fdesc |
250 | = make_fdesc (ip, gp: map->l_info[DT_PLTGOT]->d_un.d_ptr); |
251 | |
252 | if (__builtin_expect (COMPARE_AND_SWAP (&ftab[symidx], (ElfW(Addr)) NULL, |
253 | fdesc), 1)) |
254 | { |
255 | /* Noone has updated the entry and the new function |
256 | descriptor has been installed. */ |
257 | #if 0 |
258 | const char *strtab |
259 | = (const void *) D_PTR (map, l_info[DT_STRTAB]); |
260 | |
261 | ELF_MACHINE_LOAD_ADDRESS (l, local); |
262 | if (l->root != &l->boot_table |
263 | || l->boot_table.first_unused > 20) |
264 | _dl_debug_printf ("created fdesc symbol `%s' at %lx\n" , |
265 | strtab + sym->st_name, ftab[symidx]); |
266 | #endif |
267 | break; |
268 | } |
269 | else |
270 | { |
271 | /* We created a duplicated function descriptor. We put it on |
272 | free-list. */ |
273 | struct fdesc *f = (struct fdesc *) fdesc; |
274 | |
275 | ELF_MACHINE_LOAD_ADDRESS (l, local); |
276 | |
277 | do |
278 | f->ip = (ElfW(Addr)) l->free_list; |
279 | while (! COMPARE_AND_SWAP ((ElfW(Addr) *) &l->free_list, |
280 | f->ip, fdesc)); |
281 | } |
282 | } |
283 | |
284 | return ftab[symidx]; |
285 | } |
286 | |
287 | |
288 | void |
289 | _dl_unmap (struct link_map *map) |
290 | { |
291 | ElfW(Addr) *ftab = map->l_mach.fptr_table; |
292 | struct fdesc *head = NULL, *tail = NULL; |
293 | size_t i; |
294 | |
295 | _dl_unmap_segments (map); |
296 | |
297 | if (ftab == NULL) |
298 | return; |
299 | |
300 | /* String together the fdesc structures that are being freed. */ |
301 | for (i = 0; i < map->l_mach.fptr_table_len; ++i) |
302 | { |
303 | if (ftab[i]) |
304 | { |
305 | *(struct fdesc **) ftab[i] = head; |
306 | head = (struct fdesc *) ftab[i]; |
307 | if (tail == NULL) |
308 | tail = head; |
309 | } |
310 | } |
311 | |
312 | /* Prepend the new list to the free_list: */ |
313 | if (tail) |
314 | do |
315 | tail->ip = (ElfW(Addr)) local.free_list; |
316 | while (! COMPARE_AND_SWAP ((ElfW(Addr) *) &local.free_list, |
317 | tail->ip, (ElfW(Addr)) head)); |
318 | |
319 | __munmap (ftab, (map->l_mach.fptr_table_len |
320 | * sizeof (map->l_mach.fptr_table[0]))); |
321 | |
322 | map->l_mach.fptr_table = NULL; |
323 | } |
324 | |
325 | extern ElfW(Addr) _dl_fixup (struct link_map *, ElfW(Word)) attribute_hidden; |
326 | |
327 | static inline Elf32_Addr |
328 | elf_machine_resolve (void) |
329 | { |
330 | Elf32_Addr addr; |
331 | |
332 | asm ("b,l 1f,%0\n" |
333 | " addil L'_dl_runtime_resolve - ($PIC_pcrel$0 - 1),%0\n" |
334 | "1: ldo R'_dl_runtime_resolve - ($PIC_pcrel$0 - 5)(%%r1),%0\n" |
335 | : "=r" (addr) : : "r1" ); |
336 | |
337 | return addr; |
338 | } |
339 | |
340 | static inline int |
341 | _dl_read_access_allowed (unsigned int *addr) |
342 | { |
343 | int result; |
344 | |
345 | asm ("proberi (%1),3,%0" : "=r" (result) : "r" (addr) : ); |
346 | |
347 | return result; |
348 | } |
349 | |
350 | ElfW(Addr) |
351 | _dl_lookup_address (const void *address) |
352 | { |
353 | ElfW(Addr) addr = (ElfW(Addr)) address; |
354 | ElfW(Word) reloc_arg; |
355 | unsigned int *desc, *gptr; |
356 | |
357 | /* Return ADDR if the least-significant two bits of ADDR are not consistent |
358 | with ADDR being a linker defined function pointer. The normal value for |
359 | a code address in a backtrace is 3. */ |
360 | if (((uintptr_t) addr & 3) != 2) |
361 | return addr; |
362 | |
363 | /* Handle special case where ADDR points to page 0. */ |
364 | if ((uintptr_t) addr < 4096) |
365 | return addr; |
366 | |
367 | /* Clear least-significant two bits from descriptor address. */ |
368 | desc = (unsigned int *) ((uintptr_t) addr & ~3); |
369 | if (!_dl_read_access_allowed (addr: desc)) |
370 | return addr; |
371 | |
372 | /* First load the relocation offset. */ |
373 | reloc_arg = (ElfW(Word)) desc[1]; |
374 | atomic_full_barrier(); |
375 | |
376 | /* Then load first word of candidate descriptor. It should be a pointer |
377 | with word alignment and point to memory that can be read. */ |
378 | gptr = (unsigned int *) desc[0]; |
379 | if (((uintptr_t) gptr & 3) != 0 |
380 | || !_dl_read_access_allowed (addr: gptr)) |
381 | return addr; |
382 | |
383 | /* See if descriptor requires resolution. The following trampoline is |
384 | used in each global offset table for function resolution: |
385 | |
386 | ldw 0(r20),r21 |
387 | bv r0(r21) |
388 | ldw 4(r20),r21 |
389 | tramp: b,l .-12,r20 |
390 | depwi 0,31,2,r20 |
391 | .word _dl_runtime_resolve |
392 | .word "_dl_runtime_resolve ltp" |
393 | got: .word _DYNAMIC |
394 | .word "struct link map address" */ |
395 | if (gptr[0] == 0xea9f1fdd /* b,l .-12,r20 */ |
396 | && gptr[1] == 0xd6801c1e /* depwi 0,31,2,r20 */ |
397 | && (ElfW(Addr)) gptr[2] == elf_machine_resolve ()) |
398 | { |
399 | struct link_map *l = (struct link_map *) gptr[5]; |
400 | |
401 | /* If gp has been resolved, we need to hunt for relocation offset. */ |
402 | if (!(reloc_arg & PA_GP_RELOC)) |
403 | reloc_arg = _dl_fix_reloc_arg ((struct fdesc *) addr, l); |
404 | |
405 | _dl_fixup (l, reloc_arg); |
406 | } |
407 | |
408 | return (ElfW(Addr)) desc[0]; |
409 | } |
410 | rtld_hidden_def (_dl_lookup_address) |
411 | |