1 | // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) |
2 | /* Copyright (C) 2016-2022 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. |
3 | * |
4 | * SipHash: a fast short-input PRF |
5 | * https://131002.net/siphash/ |
6 | * |
7 | * This implementation is specifically for SipHash2-4 for a secure PRF |
8 | * and HalfSipHash1-3/SipHash1-3 for an insecure PRF only suitable for |
9 | * hashtables. |
10 | */ |
11 | |
12 | #include <linux/siphash.h> |
13 | #include <asm/unaligned.h> |
14 | |
15 | #if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64 |
16 | #include <linux/dcache.h> |
17 | #include <asm/word-at-a-time.h> |
18 | #endif |
19 | |
20 | #define SIPROUND SIPHASH_PERMUTATION(v0, v1, v2, v3) |
21 | |
22 | #define PREAMBLE(len) \ |
23 | u64 v0 = SIPHASH_CONST_0; \ |
24 | u64 v1 = SIPHASH_CONST_1; \ |
25 | u64 v2 = SIPHASH_CONST_2; \ |
26 | u64 v3 = SIPHASH_CONST_3; \ |
27 | u64 b = ((u64)(len)) << 56; \ |
28 | v3 ^= key->key[1]; \ |
29 | v2 ^= key->key[0]; \ |
30 | v1 ^= key->key[1]; \ |
31 | v0 ^= key->key[0]; |
32 | |
33 | #define POSTAMBLE \ |
34 | v3 ^= b; \ |
35 | SIPROUND; \ |
36 | SIPROUND; \ |
37 | v0 ^= b; \ |
38 | v2 ^= 0xff; \ |
39 | SIPROUND; \ |
40 | SIPROUND; \ |
41 | SIPROUND; \ |
42 | SIPROUND; \ |
43 | return (v0 ^ v1) ^ (v2 ^ v3); |
44 | |
45 | #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS |
46 | u64 __siphash_aligned(const void *data, size_t len, const siphash_key_t *key) |
47 | { |
48 | const u8 *end = data + len - (len % sizeof(u64)); |
49 | const u8 left = len & (sizeof(u64) - 1); |
50 | u64 m; |
51 | PREAMBLE(len) |
52 | for (; data != end; data += sizeof(u64)) { |
53 | m = le64_to_cpup(data); |
54 | v3 ^= m; |
55 | SIPROUND; |
56 | SIPROUND; |
57 | v0 ^= m; |
58 | } |
59 | #if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64 |
60 | if (left) |
61 | b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) & |
62 | bytemask_from_count(left))); |
63 | #else |
64 | switch (left) { |
65 | case 7: b |= ((u64)end[6]) << 48; fallthrough; |
66 | case 6: b |= ((u64)end[5]) << 40; fallthrough; |
67 | case 5: b |= ((u64)end[4]) << 32; fallthrough; |
68 | case 4: b |= le32_to_cpup(data); break; |
69 | case 3: b |= ((u64)end[2]) << 16; fallthrough; |
70 | case 2: b |= le16_to_cpup(data); break; |
71 | case 1: b |= end[0]; |
72 | } |
73 | #endif |
74 | POSTAMBLE |
75 | } |
76 | EXPORT_SYMBOL(__siphash_aligned); |
77 | #endif |
78 | |
79 | u64 __siphash_unaligned(const void *data, size_t len, const siphash_key_t *key) |
80 | { |
81 | const u8 *end = data + len - (len % sizeof(u64)); |
82 | const u8 left = len & (sizeof(u64) - 1); |
83 | u64 m; |
84 | PREAMBLE(len) |
85 | for (; data != end; data += sizeof(u64)) { |
86 | m = get_unaligned_le64(p: data); |
87 | v3 ^= m; |
88 | SIPROUND; |
89 | SIPROUND; |
90 | v0 ^= m; |
91 | } |
92 | #if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64 |
93 | if (left) |
94 | b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) & |
95 | bytemask_from_count(left))); |
96 | #else |
97 | switch (left) { |
98 | case 7: b |= ((u64)end[6]) << 48; fallthrough; |
99 | case 6: b |= ((u64)end[5]) << 40; fallthrough; |
100 | case 5: b |= ((u64)end[4]) << 32; fallthrough; |
101 | case 4: b |= get_unaligned_le32(end); break; |
102 | case 3: b |= ((u64)end[2]) << 16; fallthrough; |
103 | case 2: b |= get_unaligned_le16(end); break; |
104 | case 1: b |= end[0]; |
105 | } |
106 | #endif |
107 | POSTAMBLE |
108 | } |
109 | EXPORT_SYMBOL(__siphash_unaligned); |
110 | |
111 | /** |
112 | * siphash_1u64 - compute 64-bit siphash PRF value of a u64 |
113 | * @first: first u64 |
114 | * @key: the siphash key |
115 | */ |
116 | u64 siphash_1u64(const u64 first, const siphash_key_t *key) |
117 | { |
118 | PREAMBLE(8) |
119 | v3 ^= first; |
120 | SIPROUND; |
121 | SIPROUND; |
122 | v0 ^= first; |
123 | POSTAMBLE |
124 | } |
125 | EXPORT_SYMBOL(siphash_1u64); |
126 | |
127 | /** |
128 | * siphash_2u64 - compute 64-bit siphash PRF value of 2 u64 |
129 | * @first: first u64 |
130 | * @second: second u64 |
131 | * @key: the siphash key |
132 | */ |
133 | u64 siphash_2u64(const u64 first, const u64 second, const siphash_key_t *key) |
134 | { |
135 | PREAMBLE(16) |
136 | v3 ^= first; |
137 | SIPROUND; |
138 | SIPROUND; |
139 | v0 ^= first; |
140 | v3 ^= second; |
141 | SIPROUND; |
142 | SIPROUND; |
143 | v0 ^= second; |
144 | POSTAMBLE |
145 | } |
146 | EXPORT_SYMBOL(siphash_2u64); |
147 | |
148 | /** |
149 | * siphash_3u64 - compute 64-bit siphash PRF value of 3 u64 |
150 | * @first: first u64 |
151 | * @second: second u64 |
152 | * @third: third u64 |
153 | * @key: the siphash key |
154 | */ |
155 | u64 siphash_3u64(const u64 first, const u64 second, const u64 third, |
156 | const siphash_key_t *key) |
157 | { |
158 | PREAMBLE(24) |
159 | v3 ^= first; |
160 | SIPROUND; |
161 | SIPROUND; |
162 | v0 ^= first; |
163 | v3 ^= second; |
164 | SIPROUND; |
165 | SIPROUND; |
166 | v0 ^= second; |
167 | v3 ^= third; |
168 | SIPROUND; |
169 | SIPROUND; |
170 | v0 ^= third; |
171 | POSTAMBLE |
172 | } |
173 | EXPORT_SYMBOL(siphash_3u64); |
174 | |
175 | /** |
176 | * siphash_4u64 - compute 64-bit siphash PRF value of 4 u64 |
177 | * @first: first u64 |
178 | * @second: second u64 |
179 | * @third: third u64 |
180 | * @forth: forth u64 |
181 | * @key: the siphash key |
182 | */ |
183 | u64 siphash_4u64(const u64 first, const u64 second, const u64 third, |
184 | const u64 forth, const siphash_key_t *key) |
185 | { |
186 | PREAMBLE(32) |
187 | v3 ^= first; |
188 | SIPROUND; |
189 | SIPROUND; |
190 | v0 ^= first; |
191 | v3 ^= second; |
192 | SIPROUND; |
193 | SIPROUND; |
194 | v0 ^= second; |
195 | v3 ^= third; |
196 | SIPROUND; |
197 | SIPROUND; |
198 | v0 ^= third; |
199 | v3 ^= forth; |
200 | SIPROUND; |
201 | SIPROUND; |
202 | v0 ^= forth; |
203 | POSTAMBLE |
204 | } |
205 | EXPORT_SYMBOL(siphash_4u64); |
206 | |
207 | u64 siphash_1u32(const u32 first, const siphash_key_t *key) |
208 | { |
209 | PREAMBLE(4) |
210 | b |= first; |
211 | POSTAMBLE |
212 | } |
213 | EXPORT_SYMBOL(siphash_1u32); |
214 | |
215 | u64 siphash_3u32(const u32 first, const u32 second, const u32 third, |
216 | const siphash_key_t *key) |
217 | { |
218 | u64 combined = (u64)second << 32 | first; |
219 | PREAMBLE(12) |
220 | v3 ^= combined; |
221 | SIPROUND; |
222 | SIPROUND; |
223 | v0 ^= combined; |
224 | b |= third; |
225 | POSTAMBLE |
226 | } |
227 | EXPORT_SYMBOL(siphash_3u32); |
228 | |
229 | #if BITS_PER_LONG == 64 |
230 | /* Note that on 64-bit, we make HalfSipHash1-3 actually be SipHash1-3, for |
231 | * performance reasons. On 32-bit, below, we actually implement HalfSipHash1-3. |
232 | */ |
233 | |
234 | #define HSIPROUND SIPROUND |
235 | #define HPREAMBLE(len) PREAMBLE(len) |
236 | #define HPOSTAMBLE \ |
237 | v3 ^= b; \ |
238 | HSIPROUND; \ |
239 | v0 ^= b; \ |
240 | v2 ^= 0xff; \ |
241 | HSIPROUND; \ |
242 | HSIPROUND; \ |
243 | HSIPROUND; \ |
244 | return (v0 ^ v1) ^ (v2 ^ v3); |
245 | |
246 | #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS |
247 | u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key) |
248 | { |
249 | const u8 *end = data + len - (len % sizeof(u64)); |
250 | const u8 left = len & (sizeof(u64) - 1); |
251 | u64 m; |
252 | HPREAMBLE(len) |
253 | for (; data != end; data += sizeof(u64)) { |
254 | m = le64_to_cpup(data); |
255 | v3 ^= m; |
256 | HSIPROUND; |
257 | v0 ^= m; |
258 | } |
259 | #if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64 |
260 | if (left) |
261 | b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) & |
262 | bytemask_from_count(left))); |
263 | #else |
264 | switch (left) { |
265 | case 7: b |= ((u64)end[6]) << 48; fallthrough; |
266 | case 6: b |= ((u64)end[5]) << 40; fallthrough; |
267 | case 5: b |= ((u64)end[4]) << 32; fallthrough; |
268 | case 4: b |= le32_to_cpup(data); break; |
269 | case 3: b |= ((u64)end[2]) << 16; fallthrough; |
270 | case 2: b |= le16_to_cpup(data); break; |
271 | case 1: b |= end[0]; |
272 | } |
273 | #endif |
274 | HPOSTAMBLE |
275 | } |
276 | EXPORT_SYMBOL(__hsiphash_aligned); |
277 | #endif |
278 | |
279 | u32 __hsiphash_unaligned(const void *data, size_t len, |
280 | const hsiphash_key_t *key) |
281 | { |
282 | const u8 *end = data + len - (len % sizeof(u64)); |
283 | const u8 left = len & (sizeof(u64) - 1); |
284 | u64 m; |
285 | HPREAMBLE(len) |
286 | for (; data != end; data += sizeof(u64)) { |
287 | m = get_unaligned_le64(p: data); |
288 | v3 ^= m; |
289 | HSIPROUND; |
290 | v0 ^= m; |
291 | } |
292 | #if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64 |
293 | if (left) |
294 | b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) & |
295 | bytemask_from_count(left))); |
296 | #else |
297 | switch (left) { |
298 | case 7: b |= ((u64)end[6]) << 48; fallthrough; |
299 | case 6: b |= ((u64)end[5]) << 40; fallthrough; |
300 | case 5: b |= ((u64)end[4]) << 32; fallthrough; |
301 | case 4: b |= get_unaligned_le32(end); break; |
302 | case 3: b |= ((u64)end[2]) << 16; fallthrough; |
303 | case 2: b |= get_unaligned_le16(end); break; |
304 | case 1: b |= end[0]; |
305 | } |
306 | #endif |
307 | HPOSTAMBLE |
308 | } |
309 | EXPORT_SYMBOL(__hsiphash_unaligned); |
310 | |
311 | /** |
312 | * hsiphash_1u32 - compute 64-bit hsiphash PRF value of a u32 |
313 | * @first: first u32 |
314 | * @key: the hsiphash key |
315 | */ |
316 | u32 hsiphash_1u32(const u32 first, const hsiphash_key_t *key) |
317 | { |
318 | HPREAMBLE(4) |
319 | b |= first; |
320 | HPOSTAMBLE |
321 | } |
322 | EXPORT_SYMBOL(hsiphash_1u32); |
323 | |
324 | /** |
325 | * hsiphash_2u32 - compute 32-bit hsiphash PRF value of 2 u32 |
326 | * @first: first u32 |
327 | * @second: second u32 |
328 | * @key: the hsiphash key |
329 | */ |
330 | u32 hsiphash_2u32(const u32 first, const u32 second, const hsiphash_key_t *key) |
331 | { |
332 | u64 combined = (u64)second << 32 | first; |
333 | HPREAMBLE(8) |
334 | v3 ^= combined; |
335 | HSIPROUND; |
336 | v0 ^= combined; |
337 | HPOSTAMBLE |
338 | } |
339 | EXPORT_SYMBOL(hsiphash_2u32); |
340 | |
341 | /** |
342 | * hsiphash_3u32 - compute 32-bit hsiphash PRF value of 3 u32 |
343 | * @first: first u32 |
344 | * @second: second u32 |
345 | * @third: third u32 |
346 | * @key: the hsiphash key |
347 | */ |
348 | u32 hsiphash_3u32(const u32 first, const u32 second, const u32 third, |
349 | const hsiphash_key_t *key) |
350 | { |
351 | u64 combined = (u64)second << 32 | first; |
352 | HPREAMBLE(12) |
353 | v3 ^= combined; |
354 | HSIPROUND; |
355 | v0 ^= combined; |
356 | b |= third; |
357 | HPOSTAMBLE |
358 | } |
359 | EXPORT_SYMBOL(hsiphash_3u32); |
360 | |
361 | /** |
362 | * hsiphash_4u32 - compute 32-bit hsiphash PRF value of 4 u32 |
363 | * @first: first u32 |
364 | * @second: second u32 |
365 | * @third: third u32 |
366 | * @forth: forth u32 |
367 | * @key: the hsiphash key |
368 | */ |
369 | u32 hsiphash_4u32(const u32 first, const u32 second, const u32 third, |
370 | const u32 forth, const hsiphash_key_t *key) |
371 | { |
372 | u64 combined = (u64)second << 32 | first; |
373 | HPREAMBLE(16) |
374 | v3 ^= combined; |
375 | HSIPROUND; |
376 | v0 ^= combined; |
377 | combined = (u64)forth << 32 | third; |
378 | v3 ^= combined; |
379 | HSIPROUND; |
380 | v0 ^= combined; |
381 | HPOSTAMBLE |
382 | } |
383 | EXPORT_SYMBOL(hsiphash_4u32); |
384 | #else |
385 | #define HSIPROUND HSIPHASH_PERMUTATION(v0, v1, v2, v3) |
386 | |
387 | #define HPREAMBLE(len) \ |
388 | u32 v0 = HSIPHASH_CONST_0; \ |
389 | u32 v1 = HSIPHASH_CONST_1; \ |
390 | u32 v2 = HSIPHASH_CONST_2; \ |
391 | u32 v3 = HSIPHASH_CONST_3; \ |
392 | u32 b = ((u32)(len)) << 24; \ |
393 | v3 ^= key->key[1]; \ |
394 | v2 ^= key->key[0]; \ |
395 | v1 ^= key->key[1]; \ |
396 | v0 ^= key->key[0]; |
397 | |
398 | #define HPOSTAMBLE \ |
399 | v3 ^= b; \ |
400 | HSIPROUND; \ |
401 | v0 ^= b; \ |
402 | v2 ^= 0xff; \ |
403 | HSIPROUND; \ |
404 | HSIPROUND; \ |
405 | HSIPROUND; \ |
406 | return v1 ^ v3; |
407 | |
408 | #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS |
409 | u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key) |
410 | { |
411 | const u8 *end = data + len - (len % sizeof(u32)); |
412 | const u8 left = len & (sizeof(u32) - 1); |
413 | u32 m; |
414 | HPREAMBLE(len) |
415 | for (; data != end; data += sizeof(u32)) { |
416 | m = le32_to_cpup(data); |
417 | v3 ^= m; |
418 | HSIPROUND; |
419 | v0 ^= m; |
420 | } |
421 | switch (left) { |
422 | case 3: b |= ((u32)end[2]) << 16; fallthrough; |
423 | case 2: b |= le16_to_cpup(data); break; |
424 | case 1: b |= end[0]; |
425 | } |
426 | HPOSTAMBLE |
427 | } |
428 | EXPORT_SYMBOL(__hsiphash_aligned); |
429 | #endif |
430 | |
431 | u32 __hsiphash_unaligned(const void *data, size_t len, |
432 | const hsiphash_key_t *key) |
433 | { |
434 | const u8 *end = data + len - (len % sizeof(u32)); |
435 | const u8 left = len & (sizeof(u32) - 1); |
436 | u32 m; |
437 | HPREAMBLE(len) |
438 | for (; data != end; data += sizeof(u32)) { |
439 | m = get_unaligned_le32(data); |
440 | v3 ^= m; |
441 | HSIPROUND; |
442 | v0 ^= m; |
443 | } |
444 | switch (left) { |
445 | case 3: b |= ((u32)end[2]) << 16; fallthrough; |
446 | case 2: b |= get_unaligned_le16(end); break; |
447 | case 1: b |= end[0]; |
448 | } |
449 | HPOSTAMBLE |
450 | } |
451 | EXPORT_SYMBOL(__hsiphash_unaligned); |
452 | |
453 | /** |
454 | * hsiphash_1u32 - compute 32-bit hsiphash PRF value of a u32 |
455 | * @first: first u32 |
456 | * @key: the hsiphash key |
457 | */ |
458 | u32 hsiphash_1u32(const u32 first, const hsiphash_key_t *key) |
459 | { |
460 | HPREAMBLE(4) |
461 | v3 ^= first; |
462 | HSIPROUND; |
463 | v0 ^= first; |
464 | HPOSTAMBLE |
465 | } |
466 | EXPORT_SYMBOL(hsiphash_1u32); |
467 | |
468 | /** |
469 | * hsiphash_2u32 - compute 32-bit hsiphash PRF value of 2 u32 |
470 | * @first: first u32 |
471 | * @second: second u32 |
472 | * @key: the hsiphash key |
473 | */ |
474 | u32 hsiphash_2u32(const u32 first, const u32 second, const hsiphash_key_t *key) |
475 | { |
476 | HPREAMBLE(8) |
477 | v3 ^= first; |
478 | HSIPROUND; |
479 | v0 ^= first; |
480 | v3 ^= second; |
481 | HSIPROUND; |
482 | v0 ^= second; |
483 | HPOSTAMBLE |
484 | } |
485 | EXPORT_SYMBOL(hsiphash_2u32); |
486 | |
487 | /** |
488 | * hsiphash_3u32 - compute 32-bit hsiphash PRF value of 3 u32 |
489 | * @first: first u32 |
490 | * @second: second u32 |
491 | * @third: third u32 |
492 | * @key: the hsiphash key |
493 | */ |
494 | u32 hsiphash_3u32(const u32 first, const u32 second, const u32 third, |
495 | const hsiphash_key_t *key) |
496 | { |
497 | HPREAMBLE(12) |
498 | v3 ^= first; |
499 | HSIPROUND; |
500 | v0 ^= first; |
501 | v3 ^= second; |
502 | HSIPROUND; |
503 | v0 ^= second; |
504 | v3 ^= third; |
505 | HSIPROUND; |
506 | v0 ^= third; |
507 | HPOSTAMBLE |
508 | } |
509 | EXPORT_SYMBOL(hsiphash_3u32); |
510 | |
511 | /** |
512 | * hsiphash_4u32 - compute 32-bit hsiphash PRF value of 4 u32 |
513 | * @first: first u32 |
514 | * @second: second u32 |
515 | * @third: third u32 |
516 | * @forth: forth u32 |
517 | * @key: the hsiphash key |
518 | */ |
519 | u32 hsiphash_4u32(const u32 first, const u32 second, const u32 third, |
520 | const u32 forth, const hsiphash_key_t *key) |
521 | { |
522 | HPREAMBLE(16) |
523 | v3 ^= first; |
524 | HSIPROUND; |
525 | v0 ^= first; |
526 | v3 ^= second; |
527 | HSIPROUND; |
528 | v0 ^= second; |
529 | v3 ^= third; |
530 | HSIPROUND; |
531 | v0 ^= third; |
532 | v3 ^= forth; |
533 | HSIPROUND; |
534 | v0 ^= forth; |
535 | HPOSTAMBLE |
536 | } |
537 | EXPORT_SYMBOL(hsiphash_4u32); |
538 | #endif |
539 | |