1 | //===-- lib/fp_lib.h - Floating-point utilities -------------------*- C -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file is a configuration header for soft-float routines in compiler-rt. |
10 | // This file does not provide any part of the compiler-rt interface, but defines |
11 | // many useful constants and utility routines that are used in the |
12 | // implementation of the soft-float routines in compiler-rt. |
13 | // |
14 | // Assumes that float, double and long double correspond to the IEEE-754 |
15 | // binary32, binary64 and binary 128 types, respectively, and that integer |
16 | // endianness matches floating point endianness on the target platform. |
17 | // |
18 | //===----------------------------------------------------------------------===// |
19 | |
20 | #ifndef FP_LIB_HEADER |
21 | #define |
22 | |
23 | #include "int_lib.h" |
24 | #include "int_math.h" |
25 | #include <limits.h> |
26 | #include <stdbool.h> |
27 | #include <stdint.h> |
28 | |
29 | #if defined SINGLE_PRECISION |
30 | |
31 | typedef uint16_t half_rep_t; |
32 | typedef uint32_t rep_t; |
33 | typedef uint64_t twice_rep_t; |
34 | typedef int32_t srep_t; |
35 | typedef float fp_t; |
36 | #define HALF_REP_C UINT16_C |
37 | #define REP_C UINT32_C |
38 | #define significandBits 23 |
39 | |
40 | static __inline int rep_clz(rep_t a) { return clzsi(a); } |
41 | |
42 | // 32x32 --> 64 bit multiply |
43 | static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { |
44 | const uint64_t product = (uint64_t)a * b; |
45 | *hi = product >> 32; |
46 | *lo = product; |
47 | } |
48 | COMPILER_RT_ABI fp_t __addsf3(fp_t a, fp_t b); |
49 | |
50 | #elif defined DOUBLE_PRECISION |
51 | |
52 | typedef uint32_t half_rep_t; |
53 | typedef uint64_t rep_t; |
54 | typedef int64_t srep_t; |
55 | typedef double fp_t; |
56 | #define HALF_REP_C UINT32_C |
57 | #define REP_C UINT64_C |
58 | #define significandBits 52 |
59 | |
60 | static __inline int rep_clz(rep_t a) { |
61 | #if defined __LP64__ |
62 | return __builtin_clzl(a); |
63 | #else |
64 | if (a & REP_C(0xffffffff00000000)) |
65 | return clzsi(a >> 32); |
66 | else |
67 | return 32 + clzsi(a & REP_C(0xffffffff)); |
68 | #endif |
69 | } |
70 | |
71 | #define loWord(a) (a & 0xffffffffU) |
72 | #define hiWord(a) (a >> 32) |
73 | |
74 | // 64x64 -> 128 wide multiply for platforms that don't have such an operation; |
75 | // many 64-bit platforms have this operation, but they tend to have hardware |
76 | // floating-point, so we don't bother with a special case for them here. |
77 | static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { |
78 | // Each of the component 32x32 -> 64 products |
79 | const uint64_t plolo = loWord(a) * loWord(b); |
80 | const uint64_t plohi = loWord(a) * hiWord(b); |
81 | const uint64_t philo = hiWord(a) * loWord(b); |
82 | const uint64_t phihi = hiWord(a) * hiWord(b); |
83 | // Sum terms that contribute to lo in a way that allows us to get the carry |
84 | const uint64_t r0 = loWord(plolo); |
85 | const uint64_t r1 = hiWord(plolo) + loWord(plohi) + loWord(philo); |
86 | *lo = r0 + (r1 << 32); |
87 | // Sum terms contributing to hi with the carry from lo |
88 | *hi = hiWord(plohi) + hiWord(philo) + hiWord(r1) + phihi; |
89 | } |
90 | #undef loWord |
91 | #undef hiWord |
92 | |
93 | COMPILER_RT_ABI fp_t __adddf3(fp_t a, fp_t b); |
94 | |
95 | #elif defined QUAD_PRECISION |
96 | #if defined(CRT_HAS_TF_MODE) |
97 | typedef uint64_t half_rep_t; |
98 | typedef __uint128_t rep_t; |
99 | typedef __int128_t srep_t; |
100 | typedef tf_float fp_t; |
101 | #define HALF_REP_C UINT64_C |
102 | #define REP_C (__uint128_t) |
103 | // Note: Since there is no explicit way to tell compiler the constant is a |
104 | // 128-bit integer, we let the constant be casted to 128-bit integer |
105 | #define significandBits 112 |
106 | #define TF_MANT_DIG (significandBits + 1) |
107 | |
108 | static __inline int rep_clz(rep_t a) { |
109 | const union { |
110 | __uint128_t ll; |
111 | #if _YUGA_BIG_ENDIAN |
112 | struct { |
113 | uint64_t high, low; |
114 | } s; |
115 | #else |
116 | struct { |
117 | uint64_t low, high; |
118 | } s; |
119 | #endif |
120 | } uu = {.ll = a}; |
121 | |
122 | uint64_t word; |
123 | uint64_t add; |
124 | |
125 | if (uu.s.high) { |
126 | word = uu.s.high; |
127 | add = 0; |
128 | } else { |
129 | word = uu.s.low; |
130 | add = 64; |
131 | } |
132 | return __builtin_clzll(word) + add; |
133 | } |
134 | |
135 | #define Word_LoMask UINT64_C(0x00000000ffffffff) |
136 | #define Word_HiMask UINT64_C(0xffffffff00000000) |
137 | #define Word_FullMask UINT64_C(0xffffffffffffffff) |
138 | #define Word_1(a) (uint64_t)((a >> 96) & Word_LoMask) |
139 | #define Word_2(a) (uint64_t)((a >> 64) & Word_LoMask) |
140 | #define Word_3(a) (uint64_t)((a >> 32) & Word_LoMask) |
141 | #define Word_4(a) (uint64_t)(a & Word_LoMask) |
142 | |
143 | // 128x128 -> 256 wide multiply for platforms that don't have such an operation; |
144 | // many 64-bit platforms have this operation, but they tend to have hardware |
145 | // floating-point, so we don't bother with a special case for them here. |
146 | static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { |
147 | |
148 | const uint64_t product11 = Word_1(a) * Word_1(b); |
149 | const uint64_t product12 = Word_1(a) * Word_2(b); |
150 | const uint64_t product13 = Word_1(a) * Word_3(b); |
151 | const uint64_t product14 = Word_1(a) * Word_4(b); |
152 | const uint64_t product21 = Word_2(a) * Word_1(b); |
153 | const uint64_t product22 = Word_2(a) * Word_2(b); |
154 | const uint64_t product23 = Word_2(a) * Word_3(b); |
155 | const uint64_t product24 = Word_2(a) * Word_4(b); |
156 | const uint64_t product31 = Word_3(a) * Word_1(b); |
157 | const uint64_t product32 = Word_3(a) * Word_2(b); |
158 | const uint64_t product33 = Word_3(a) * Word_3(b); |
159 | const uint64_t product34 = Word_3(a) * Word_4(b); |
160 | const uint64_t product41 = Word_4(a) * Word_1(b); |
161 | const uint64_t product42 = Word_4(a) * Word_2(b); |
162 | const uint64_t product43 = Word_4(a) * Word_3(b); |
163 | const uint64_t product44 = Word_4(a) * Word_4(b); |
164 | |
165 | const __uint128_t sum0 = (__uint128_t)product44; |
166 | const __uint128_t sum1 = (__uint128_t)product34 + (__uint128_t)product43; |
167 | const __uint128_t sum2 = |
168 | (__uint128_t)product24 + (__uint128_t)product33 + (__uint128_t)product42; |
169 | const __uint128_t sum3 = (__uint128_t)product14 + (__uint128_t)product23 + |
170 | (__uint128_t)product32 + (__uint128_t)product41; |
171 | const __uint128_t sum4 = |
172 | (__uint128_t)product13 + (__uint128_t)product22 + (__uint128_t)product31; |
173 | const __uint128_t sum5 = (__uint128_t)product12 + (__uint128_t)product21; |
174 | const __uint128_t sum6 = (__uint128_t)product11; |
175 | |
176 | const __uint128_t r0 = (sum0 & Word_FullMask) + ((sum1 & Word_LoMask) << 32); |
177 | const __uint128_t r1 = (sum0 >> 64) + ((sum1 >> 32) & Word_FullMask) + |
178 | (sum2 & Word_FullMask) + ((sum3 << 32) & Word_HiMask); |
179 | |
180 | *lo = r0 + (r1 << 64); |
181 | *hi = (r1 >> 64) + (sum1 >> 96) + (sum2 >> 64) + (sum3 >> 32) + sum4 + |
182 | (sum5 << 32) + (sum6 << 64); |
183 | } |
184 | #undef Word_1 |
185 | #undef Word_2 |
186 | #undef Word_3 |
187 | #undef Word_4 |
188 | #undef Word_HiMask |
189 | #undef Word_LoMask |
190 | #undef Word_FullMask |
191 | #endif // defined(CRT_HAS_TF_MODE) |
192 | #else |
193 | #error SINGLE_PRECISION, DOUBLE_PRECISION or QUAD_PRECISION must be defined. |
194 | #endif |
195 | |
196 | #if defined(SINGLE_PRECISION) || defined(DOUBLE_PRECISION) || \ |
197 | (defined(QUAD_PRECISION) && defined(CRT_HAS_TF_MODE)) |
198 | #define typeWidth (sizeof(rep_t) * CHAR_BIT) |
199 | #define exponentBits (typeWidth - significandBits - 1) |
200 | #define maxExponent ((1 << exponentBits) - 1) |
201 | #define exponentBias (maxExponent >> 1) |
202 | |
203 | #define implicitBit (REP_C(1) << significandBits) |
204 | #define significandMask (implicitBit - 1U) |
205 | #define signBit (REP_C(1) << (significandBits + exponentBits)) |
206 | #define absMask (signBit - 1U) |
207 | #define exponentMask (absMask ^ significandMask) |
208 | #define oneRep ((rep_t)exponentBias << significandBits) |
209 | #define infRep exponentMask |
210 | #define quietBit (implicitBit >> 1) |
211 | #define qnanRep (exponentMask | quietBit) |
212 | |
213 | static __inline rep_t toRep(fp_t x) { |
214 | const union { |
215 | fp_t f; |
216 | rep_t i; |
217 | } rep = {.f = x}; |
218 | return rep.i; |
219 | } |
220 | |
221 | static __inline fp_t fromRep(rep_t x) { |
222 | const union { |
223 | fp_t f; |
224 | rep_t i; |
225 | } rep = {.i = x}; |
226 | return rep.f; |
227 | } |
228 | |
229 | static __inline int normalize(rep_t *significand) { |
230 | const int shift = rep_clz(a: *significand) - rep_clz(implicitBit); |
231 | *significand <<= shift; |
232 | return 1 - shift; |
233 | } |
234 | |
235 | static __inline void wideLeftShift(rep_t *hi, rep_t *lo, int count) { |
236 | *hi = *hi << count | *lo >> (typeWidth - count); |
237 | *lo = *lo << count; |
238 | } |
239 | |
240 | static __inline void wideRightShiftWithSticky(rep_t *hi, rep_t *lo, |
241 | unsigned int count) { |
242 | if (count < typeWidth) { |
243 | const bool sticky = (*lo << (typeWidth - count)) != 0; |
244 | *lo = *hi << (typeWidth - count) | *lo >> count | sticky; |
245 | *hi = *hi >> count; |
246 | } else if (count < 2 * typeWidth) { |
247 | const bool sticky = *hi << (2 * typeWidth - count) | *lo; |
248 | *lo = *hi >> (count - typeWidth) | sticky; |
249 | *hi = 0; |
250 | } else { |
251 | const bool sticky = *hi | *lo; |
252 | *lo = sticky; |
253 | *hi = 0; |
254 | } |
255 | } |
256 | |
257 | // Implements logb methods (logb, logbf, logbl) for IEEE-754. This avoids |
258 | // pulling in a libm dependency from compiler-rt, but is not meant to replace |
259 | // it (i.e. code calling logb() should get the one from libm, not this), hence |
260 | // the __compiler_rt prefix. |
261 | static __inline fp_t __compiler_rt_logbX(fp_t x) { |
262 | rep_t rep = toRep(x); |
263 | int exp = (rep & exponentMask) >> significandBits; |
264 | |
265 | // Abnormal cases: |
266 | // 1) +/- inf returns +inf; NaN returns NaN |
267 | // 2) 0.0 returns -inf |
268 | if (exp == maxExponent) { |
269 | if (((rep & signBit) == 0) || (x != x)) { |
270 | return x; // NaN or +inf: return x |
271 | } else { |
272 | return -x; // -inf: return -x |
273 | } |
274 | } else if (x == 0.0) { |
275 | // 0.0: return -inf |
276 | return fromRep(infRep | signBit); |
277 | } |
278 | |
279 | if (exp != 0) { |
280 | // Normal number |
281 | return exp - exponentBias; // Unbias exponent |
282 | } else { |
283 | // Subnormal number; normalize and repeat |
284 | rep &= absMask; |
285 | const int shift = 1 - normalize(significand: &rep); |
286 | exp = (rep & exponentMask) >> significandBits; |
287 | return exp - exponentBias - shift; // Unbias exponent |
288 | } |
289 | } |
290 | |
291 | // Avoid using scalbn from libm. Unlike libc/libm scalbn, this function never |
292 | // sets errno on underflow/overflow. |
293 | static __inline fp_t __compiler_rt_scalbnX(fp_t x, int y) { |
294 | const rep_t rep = toRep(x); |
295 | int exp = (rep & exponentMask) >> significandBits; |
296 | |
297 | if (x == 0.0 || exp == maxExponent) |
298 | return x; // +/- 0.0, NaN, or inf: return x |
299 | |
300 | // Normalize subnormal input. |
301 | rep_t sig = rep & significandMask; |
302 | if (exp == 0) { |
303 | exp += normalize(significand: &sig); |
304 | sig &= ~implicitBit; // clear the implicit bit again |
305 | } |
306 | |
307 | if (__builtin_sadd_overflow(exp, y, &exp)) { |
308 | // Saturate the exponent, which will guarantee an underflow/overflow below. |
309 | exp = (y >= 0) ? INT_MAX : INT_MIN; |
310 | } |
311 | |
312 | // Return this value: [+/-] 1.sig * 2 ** (exp - exponentBias). |
313 | const rep_t sign = rep & signBit; |
314 | if (exp >= maxExponent) { |
315 | // Overflow, which could produce infinity or the largest-magnitude value, |
316 | // depending on the rounding mode. |
317 | return fromRep(x: sign | ((rep_t)(maxExponent - 1) << significandBits)) * 2.0f; |
318 | } else if (exp <= 0) { |
319 | // Subnormal or underflow. Use floating-point multiply to handle truncation |
320 | // correctly. |
321 | fp_t tmp = fromRep(x: sign | (REP_C(1) << significandBits) | sig); |
322 | exp += exponentBias - 1; |
323 | if (exp < 1) |
324 | exp = 1; |
325 | tmp *= fromRep(x: (rep_t)exp << significandBits); |
326 | return tmp; |
327 | } else |
328 | return fromRep(x: sign | ((rep_t)exp << significandBits) | sig); |
329 | } |
330 | |
331 | // Avoid using fmax from libm. |
332 | static __inline fp_t __compiler_rt_fmaxX(fp_t x, fp_t y) { |
333 | // If either argument is NaN, return the other argument. If both are NaN, |
334 | // arbitrarily return the second one. Otherwise, if both arguments are +/-0, |
335 | // arbitrarily return the first one. |
336 | return (crt_isnan(x) || x < y) ? y : x; |
337 | } |
338 | |
339 | #endif |
340 | |
341 | #if defined(SINGLE_PRECISION) |
342 | |
343 | static __inline fp_t __compiler_rt_logbf(fp_t x) { |
344 | return __compiler_rt_logbX(x); |
345 | } |
346 | static __inline fp_t __compiler_rt_scalbnf(fp_t x, int y) { |
347 | return __compiler_rt_scalbnX(x, y); |
348 | } |
349 | static __inline fp_t __compiler_rt_fmaxf(fp_t x, fp_t y) { |
350 | #if defined(__aarch64__) |
351 | // Use __builtin_fmaxf which turns into an fmaxnm instruction on AArch64. |
352 | return __builtin_fmaxf(x, y); |
353 | #else |
354 | // __builtin_fmaxf frequently turns into a libm call, so inline the function. |
355 | return __compiler_rt_fmaxX(x, y); |
356 | #endif |
357 | } |
358 | |
359 | #elif defined(DOUBLE_PRECISION) |
360 | |
361 | static __inline fp_t __compiler_rt_logb(fp_t x) { |
362 | return __compiler_rt_logbX(x); |
363 | } |
364 | static __inline fp_t __compiler_rt_scalbn(fp_t x, int y) { |
365 | return __compiler_rt_scalbnX(x, y); |
366 | } |
367 | static __inline fp_t __compiler_rt_fmax(fp_t x, fp_t y) { |
368 | #if defined(__aarch64__) |
369 | // Use __builtin_fmax which turns into an fmaxnm instruction on AArch64. |
370 | return __builtin_fmax(x, y); |
371 | #else |
372 | // __builtin_fmax frequently turns into a libm call, so inline the function. |
373 | return __compiler_rt_fmaxX(x, y); |
374 | #endif |
375 | } |
376 | |
377 | #elif defined(QUAD_PRECISION) && defined(CRT_HAS_TF_MODE) |
378 | // The generic implementation only works for ieee754 floating point. For other |
379 | // floating point types, continue to rely on the libm implementation for now. |
380 | #if defined(CRT_HAS_IEEE_TF) |
381 | static __inline tf_float __compiler_rt_logbtf(tf_float x) { |
382 | return __compiler_rt_logbX(x); |
383 | } |
384 | static __inline tf_float __compiler_rt_scalbntf(tf_float x, int y) { |
385 | return __compiler_rt_scalbnX(x, y); |
386 | } |
387 | static __inline tf_float __compiler_rt_fmaxtf(tf_float x, tf_float y) { |
388 | return __compiler_rt_fmaxX(x, y); |
389 | } |
390 | #define __compiler_rt_logbl __compiler_rt_logbtf |
391 | #define __compiler_rt_scalbnl __compiler_rt_scalbntf |
392 | #define __compiler_rt_fmaxl __compiler_rt_fmaxtf |
393 | #define crt_fabstf crt_fabsf128 |
394 | #define crt_copysigntf crt_copysignf128 |
395 | #elif defined(CRT_LDBL_128BIT) |
396 | static __inline tf_float __compiler_rt_logbtf(tf_float x) { |
397 | return crt_logbl(x); |
398 | } |
399 | static __inline tf_float __compiler_rt_scalbntf(tf_float x, int y) { |
400 | return crt_scalbnl(x, y); |
401 | } |
402 | static __inline tf_float __compiler_rt_fmaxtf(tf_float x, tf_float y) { |
403 | return crt_fmaxl(x, y); |
404 | } |
405 | #define __compiler_rt_logbl crt_logbl |
406 | #define __compiler_rt_scalbnl crt_scalbnl |
407 | #define __compiler_rt_fmaxl crt_fmaxl |
408 | #else |
409 | #error Unsupported TF mode type |
410 | #endif |
411 | |
412 | #endif // *_PRECISION |
413 | |
414 | #endif // FP_LIB_HEADER |
415 | |