1/* Hardware capability support for run-time dynamic loader.
2 Copyright (C) 2012-2022 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
18
19#include <assert.h>
20#include <elf.h>
21#include <errno.h>
22#include <libintl.h>
23#include <unistd.h>
24#include <ldsodefs.h>
25
26#include <dl-procinfo.h>
27#include <dl-hwcaps.h>
28
29/* This is the result of counting the substrings in a colon-separated
30 hwcaps string. */
31struct hwcaps_counts
32{
33 /* Number of substrings. */
34 size_t count;
35
36 /* Sum of the individual substring lengths (without separators or
37 null terminators). */
38 size_t total_length;
39
40 /* Maximum length of an individual substring. */
41 size_t maximum_length;
42};
43
44/* Update *COUNTS according to the contents of HWCAPS. Skip over
45 entries whose bit is not set in MASK. */
46static void
47update_hwcaps_counts (struct hwcaps_counts *counts, const char *hwcaps,
48 uint32_t bitmask, const char *mask)
49{
50 struct dl_hwcaps_split_masked sp;
51 _dl_hwcaps_split_masked_init (s: &sp, subject: hwcaps, bitmask, mask);
52 while (_dl_hwcaps_split_masked (s: &sp))
53 {
54 ++counts->count;
55 counts->total_length += sp.split.length;
56 if (sp.split.length > counts->maximum_length)
57 counts->maximum_length = sp.split.length;
58 }
59}
60
61/* State for copy_hwcaps. Must be initialized to point to
62 the storage areas for the array and the strings themselves. */
63struct copy_hwcaps
64{
65 struct r_strlenpair *next_pair;
66 char *next_string;
67};
68
69/* Copy HWCAPS into the string pairs and strings, advancing *TARGET.
70 Skip over entries whose bit is not set in MASK. */
71static void
72copy_hwcaps (struct copy_hwcaps *target, const char *hwcaps,
73 uint32_t bitmask, const char *mask)
74{
75 struct dl_hwcaps_split_masked sp;
76 _dl_hwcaps_split_masked_init (s: &sp, subject: hwcaps, bitmask, mask);
77 while (_dl_hwcaps_split_masked (s: &sp))
78 {
79 target->next_pair->str = target->next_string;
80 char *slash = __mempcpy (__mempcpy (target->next_string,
81 GLIBC_HWCAPS_PREFIX,
82 strlen (GLIBC_HWCAPS_PREFIX)),
83 sp.split.segment, sp.split.length);
84 *slash = '/';
85 target->next_pair->len
86 = strlen (GLIBC_HWCAPS_PREFIX) + sp.split.length + 1;
87 ++target->next_pair;
88 target->next_string = slash + 1;
89 }
90}
91
92struct dl_hwcaps_priority *_dl_hwcaps_priorities;
93uint32_t _dl_hwcaps_priorities_length;
94
95/* Allocate _dl_hwcaps_priorities and fill it with data. */
96static void
97compute_priorities (size_t total_count, const char *prepend,
98 uint32_t bitmask, const char *mask)
99{
100 _dl_hwcaps_priorities = malloc (size: total_count
101 * sizeof (*_dl_hwcaps_priorities));
102 if (_dl_hwcaps_priorities == NULL)
103 _dl_signal_error (ENOMEM, NULL, NULL,
104 N_("cannot create HWCAP priorities"));
105 _dl_hwcaps_priorities_length = total_count;
106
107 /* First the prepended subdirectories. */
108 size_t i = 0;
109 {
110 struct dl_hwcaps_split sp;
111 _dl_hwcaps_split_init (s: &sp, subject: prepend);
112 while (_dl_hwcaps_split (s: &sp))
113 {
114 _dl_hwcaps_priorities[i].name = sp.segment;
115 _dl_hwcaps_priorities[i].name_length = sp.length;
116 _dl_hwcaps_priorities[i].priority = i + 1;
117 ++i;
118 }
119 }
120
121 /* Then the built-in subdirectories that are actually active. */
122 {
123 struct dl_hwcaps_split_masked sp;
124 _dl_hwcaps_split_masked_init (s: &sp, subject: _dl_hwcaps_subdirs, bitmask, mask);
125 while (_dl_hwcaps_split_masked (s: &sp))
126 {
127 _dl_hwcaps_priorities[i].name = sp.split.segment;
128 _dl_hwcaps_priorities[i].name_length = sp.split.length;
129 _dl_hwcaps_priorities[i].priority = i + 1;
130 ++i;
131 }
132 }
133 assert (i == total_count);
134}
135
136/* Sort the _dl_hwcaps_priorities array by name. */
137static void
138sort_priorities_by_name (void)
139{
140 /* Insertion sort. There is no need to link qsort into the dynamic
141 loader for such a short array. */
142 for (size_t i = 1; i < _dl_hwcaps_priorities_length; ++i)
143 for (size_t j = i; j > 0; --j)
144 {
145 struct dl_hwcaps_priority *previous = _dl_hwcaps_priorities + j - 1;
146 struct dl_hwcaps_priority *current = _dl_hwcaps_priorities + j;
147
148 /* Bail out if current is greater or equal to the previous
149 value. */
150 uint32_t to_compare;
151 if (current->name_length < previous->name_length)
152 to_compare = current->name_length;
153 else
154 to_compare = previous->name_length;
155 int cmp = memcmp (current->name, previous->name, to_compare);
156 if (cmp > 0
157 || (cmp == 0 && current->name_length >= previous->name_length))
158 break;
159
160 /* Swap *previous and *current. */
161 struct dl_hwcaps_priority tmp = *previous;
162 *previous = *current;
163 *current = tmp;
164 }
165}
166
167/* Return an array of useful/necessary hardware capability names. */
168const struct r_strlenpair *
169_dl_important_hwcaps (const char *glibc_hwcaps_prepend,
170 const char *glibc_hwcaps_mask,
171 size_t *sz, size_t *max_capstrlen)
172{
173 uint64_t hwcap_mask = GET_HWCAP_MASK();
174 /* Determine how many important bits are set. */
175 uint64_t masked = GLRO(dl_hwcap) & hwcap_mask;
176 size_t cnt = GLRO (dl_platform) != NULL;
177 size_t n, m;
178 struct r_strlenpair *result;
179 struct r_strlenpair *rp;
180 char *cp;
181
182 /* glibc-hwcaps subdirectories. These are exempted from the power
183 set construction below. */
184 uint32_t hwcaps_subdirs_active = _dl_hwcaps_subdirs_active ();
185 struct hwcaps_counts hwcaps_counts = { 0, };
186 update_hwcaps_counts (counts: &hwcaps_counts, hwcaps: glibc_hwcaps_prepend, bitmask: -1, NULL);
187 update_hwcaps_counts (counts: &hwcaps_counts, hwcaps: _dl_hwcaps_subdirs,
188 bitmask: hwcaps_subdirs_active, mask: glibc_hwcaps_mask);
189 compute_priorities (total_count: hwcaps_counts.count, prepend: glibc_hwcaps_prepend,
190 bitmask: hwcaps_subdirs_active, mask: glibc_hwcaps_mask);
191 sort_priorities_by_name ();
192
193 /* Each hwcaps subdirectory has a GLIBC_HWCAPS_PREFIX string prefix
194 and a "/" suffix once stored in the result. */
195 hwcaps_counts.maximum_length += strlen (GLIBC_HWCAPS_PREFIX) + 1;
196 size_t hwcaps_sz = (hwcaps_counts.count * (strlen (GLIBC_HWCAPS_PREFIX) + 1)
197 + hwcaps_counts.total_length);
198
199 /* Count the number of bits set in the masked value. */
200 for (n = 0; (~((1ULL << n) - 1) & masked) != 0; ++n)
201 if ((masked & (1ULL << n)) != 0)
202 ++cnt;
203
204 /* For TLS enabled builds always add 'tls'. */
205 ++cnt;
206
207 /* Create temporary data structure to generate result table. */
208 struct r_strlenpair temp[cnt];
209 m = 0;
210 for (n = 0; masked != 0; ++n)
211 if ((masked & (1ULL << n)) != 0)
212 {
213 temp[m].str = _dl_hwcap_string (idx: n);
214 temp[m].len = strlen (temp[m].str);
215 masked ^= 1ULL << n;
216 ++m;
217 }
218 if (GLRO (dl_platform) != NULL)
219 {
220 temp[m].str = GLRO (dl_platform);
221 temp[m].len = GLRO (dl_platformlen);
222 ++m;
223 }
224
225 temp[m].str = "tls";
226 temp[m].len = 3;
227 ++m;
228
229 assert (m == cnt);
230
231 /* Determine the total size of all strings together. */
232 size_t total;
233 if (cnt == 1)
234 total = temp[0].len + 1;
235 else
236 {
237 total = temp[0].len + temp[cnt - 1].len + 2;
238 if (cnt > 2)
239 {
240 total <<= 1;
241 for (n = 1; n + 1 < cnt; ++n)
242 total += temp[n].len + 1;
243 if (cnt > 3
244 && (cnt >= sizeof (size_t) * 8
245 || total + (sizeof (*result) << 3)
246 >= (1UL << (sizeof (size_t) * 8 - cnt + 3))))
247 _dl_signal_error (ENOMEM, NULL, NULL,
248 N_("cannot create capability list"));
249
250 total <<= cnt - 3;
251 }
252 }
253
254 *sz = hwcaps_counts.count + (1 << cnt);
255
256 /* This is the overall result, including both glibc-hwcaps
257 subdirectories and the legacy hwcaps subdirectories using the
258 power set construction. */
259 total += hwcaps_sz;
260 struct r_strlenpair *overall_result
261 = malloc (size: *sz * sizeof (*result) + total);
262 if (overall_result == NULL)
263 _dl_signal_error (ENOMEM, NULL, NULL,
264 N_("cannot create capability list"));
265
266 /* Fill in the glibc-hwcaps subdirectories. */
267 {
268 struct copy_hwcaps target;
269 target.next_pair = overall_result;
270 target.next_string = (char *) (overall_result + *sz);
271 copy_hwcaps (target: &target, hwcaps: glibc_hwcaps_prepend, bitmask: -1, NULL);
272 copy_hwcaps (target: &target, hwcaps: _dl_hwcaps_subdirs,
273 bitmask: hwcaps_subdirs_active, mask: glibc_hwcaps_mask);
274 /* Set up the write target for the power set construction. */
275 result = target.next_pair;
276 cp = target.next_string;
277 }
278
279
280 /* Power set construction begins here. We use a very compressed way
281 to store the various combinations of capability names. */
282
283 if (cnt == 1)
284 {
285 result[0].str = cp;
286 result[0].len = temp[0].len + 1;
287 result[1].str = cp;
288 result[1].len = 0;
289 cp = __mempcpy (cp, temp[0].str, temp[0].len);
290 *cp = '/';
291 if (result[0].len > hwcaps_counts.maximum_length)
292 *max_capstrlen = result[0].len;
293 else
294 *max_capstrlen = hwcaps_counts.maximum_length;
295
296 return overall_result;
297 }
298
299 /* Fill in the information. This follows the following scheme
300 (indices from TEMP for four strings):
301 entry #0: 0, 1, 2, 3 binary: 1111
302 #1: 0, 1, 3 1101
303 #2: 0, 2, 3 1011
304 #3: 0, 3 1001
305 This allows the representation of all possible combinations of
306 capability names in the string. First generate the strings. */
307 result[1].str = result[0].str = cp;
308#define add(idx) \
309 cp = __mempcpy (__mempcpy (cp, temp[idx].str, temp[idx].len), "/", 1);
310 if (cnt == 2)
311 {
312 add (1);
313 add (0);
314 }
315 else
316 {
317 n = 1 << (cnt - 1);
318 do
319 {
320 n -= 2;
321
322 /* We always add the last string. */
323 add (cnt - 1);
324
325 /* Add the strings which have the bit set in N. */
326 for (m = cnt - 2; m > 0; --m)
327 if ((n & (1 << m)) != 0)
328 add (m);
329
330 /* Always add the first string. */
331 add (0);
332 }
333 while (n != 0);
334 }
335#undef add
336
337 /* Now we are ready to install the string pointers and length. */
338 for (n = 0; n < (1UL << cnt); ++n)
339 result[n].len = 0;
340 n = cnt;
341 do
342 {
343 size_t mask = 1 << --n;
344
345 rp = result;
346 for (m = 1 << cnt; m > 0; ++rp)
347 if ((--m & mask) != 0)
348 rp->len += temp[n].len + 1;
349 }
350 while (n != 0);
351
352 /* The first half of the strings all include the first string. */
353 n = (1 << cnt) - 2;
354 rp = &result[2];
355 while (n != (1UL << (cnt - 1)))
356 {
357 if ((--n & 1) != 0)
358 rp[0].str = rp[-2].str + rp[-2].len;
359 else
360 rp[0].str = rp[-1].str;
361 ++rp;
362 }
363
364 /* The second half starts right after the first part of the string of
365 the corresponding entry in the first half. */
366 do
367 {
368 rp[0].str = rp[-(1 << (cnt - 1))].str + temp[cnt - 1].len + 1;
369 ++rp;
370 }
371 while (--n != 0);
372
373 /* The maximum string length. */
374 if (result[0].len > hwcaps_counts.maximum_length)
375 *max_capstrlen = result[0].len;
376 else
377 *max_capstrlen = hwcaps_counts.maximum_length;
378
379 return overall_result;
380}
381

source code of glibc/elf/dl-hwcaps.c