1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
2 | # |
3 | # Accelerated poly1305 implementation for ppc64le. |
4 | # |
5 | # Copyright 2023- IBM Corp. All rights reserved |
6 | # |
7 | #=================================================================================== |
8 | # Written by Danny Tsen <dtsen@us.ibm.com> |
9 | # |
10 | # Poly1305 - this version mainly using vector/VSX/Scalar |
11 | # - 26 bits limbs |
12 | # - Handle multiple 64 byte blcok. |
13 | # |
14 | # Block size 16 bytes |
15 | # key = (r, s) |
16 | # clamp r &= 0x0FFFFFFC0FFFFFFC 0x0FFFFFFC0FFFFFFF |
17 | # p = 2^130 - 5 |
18 | # a += m |
19 | # a = (r + a) % p |
20 | # a += s |
21 | # |
22 | # Improve performance by breaking down polynominal to the sum of products with |
23 | # h4 = m1 * r⁴ + m2 * r³ + m3 * r² + m4 * r |
24 | # |
25 | # 07/22/21 - this revison based on the above sum of products. Setup r^4, r^3, r^2, r and s3, s2, s1, s0 |
26 | # to 9 vectors for multiplications. |
27 | # |
28 | # setup r^4, r^3, r^2, r vectors |
29 | # vs [r^1, r^3, r^2, r^4] |
30 | # vs0 = [r0,.....] |
31 | # vs1 = [r1,.....] |
32 | # vs2 = [r2,.....] |
33 | # vs3 = [r3,.....] |
34 | # vs4 = [r4,.....] |
35 | # vs5 = [r1*5,...] |
36 | # vs6 = [r2*5,...] |
37 | # vs7 = [r2*5,...] |
38 | # vs8 = [r4*5,...] |
39 | # |
40 | # Each word in a vector consists a member of a "r/s" in [a * r/s]. |
41 | # |
42 | # r0, r4*5, r3*5, r2*5, r1*5; |
43 | # r1, r0, r4*5, r3*5, r2*5; |
44 | # r2, r1, r0, r4*5, r3*5; |
45 | # r3, r2, r1, r0, r4*5; |
46 | # r4, r3, r2, r1, r0 ; |
47 | # |
48 | # |
49 | # poly1305_p10le_4blocks( uint8_t *k, uint32_t mlen, uint8_t *m) |
50 | # k = 32 bytes key |
51 | # r3 = k (r, s) |
52 | # r4 = mlen |
53 | # r5 = m |
54 | # |
55 | #include <asm/ppc_asm.h> |
56 | #include <asm/asm-offsets.h> |
57 | #include <asm/asm-compat.h> |
58 | #include <linux/linkage.h> |
59 | |
60 | .machine "any" |
61 | |
62 | .text |
63 | |
64 | .macro SAVE_GPR GPR OFFSET FRAME |
65 | std \GPR,\OFFSET(\FRAME) |
66 | .endm |
67 | |
68 | .macro SAVE_VRS VRS OFFSET FRAME |
69 | li 16, \OFFSET |
70 | stvx \VRS, 16, \FRAME |
71 | .endm |
72 | |
73 | .macro SAVE_VSX VSX OFFSET FRAME |
74 | li 16, \OFFSET |
75 | stxvx \VSX, 16, \FRAME |
76 | .endm |
77 | |
78 | .macro RESTORE_GPR GPR OFFSET FRAME |
79 | ld \GPR,\OFFSET(\FRAME) |
80 | .endm |
81 | |
82 | .macro RESTORE_VRS VRS OFFSET FRAME |
83 | li 16, \OFFSET |
84 | lvx \VRS, 16, \FRAME |
85 | .endm |
86 | |
87 | .macro RESTORE_VSX VSX OFFSET FRAME |
88 | li 16, \OFFSET |
89 | lxvx \VSX, 16, \FRAME |
90 | .endm |
91 | |
92 | .macro SAVE_REGS |
93 | mflr 0 |
94 | std 0, 16(1) |
95 | stdu 1,-752(1) |
96 | |
97 | SAVE_GPR 14, 112, 1 |
98 | SAVE_GPR 15, 120, 1 |
99 | SAVE_GPR 16, 128, 1 |
100 | SAVE_GPR 17, 136, 1 |
101 | SAVE_GPR 18, 144, 1 |
102 | SAVE_GPR 19, 152, 1 |
103 | SAVE_GPR 20, 160, 1 |
104 | SAVE_GPR 21, 168, 1 |
105 | SAVE_GPR 22, 176, 1 |
106 | SAVE_GPR 23, 184, 1 |
107 | SAVE_GPR 24, 192, 1 |
108 | SAVE_GPR 25, 200, 1 |
109 | SAVE_GPR 26, 208, 1 |
110 | SAVE_GPR 27, 216, 1 |
111 | SAVE_GPR 28, 224, 1 |
112 | SAVE_GPR 29, 232, 1 |
113 | SAVE_GPR 30, 240, 1 |
114 | SAVE_GPR 31, 248, 1 |
115 | |
116 | addi 9, 1, 256 |
117 | SAVE_VRS 20, 0, 9 |
118 | SAVE_VRS 21, 16, 9 |
119 | SAVE_VRS 22, 32, 9 |
120 | SAVE_VRS 23, 48, 9 |
121 | SAVE_VRS 24, 64, 9 |
122 | SAVE_VRS 25, 80, 9 |
123 | SAVE_VRS 26, 96, 9 |
124 | SAVE_VRS 27, 112, 9 |
125 | SAVE_VRS 28, 128, 9 |
126 | SAVE_VRS 29, 144, 9 |
127 | SAVE_VRS 30, 160, 9 |
128 | SAVE_VRS 31, 176, 9 |
129 | |
130 | SAVE_VSX 14, 192, 9 |
131 | SAVE_VSX 15, 208, 9 |
132 | SAVE_VSX 16, 224, 9 |
133 | SAVE_VSX 17, 240, 9 |
134 | SAVE_VSX 18, 256, 9 |
135 | SAVE_VSX 19, 272, 9 |
136 | SAVE_VSX 20, 288, 9 |
137 | SAVE_VSX 21, 304, 9 |
138 | SAVE_VSX 22, 320, 9 |
139 | SAVE_VSX 23, 336, 9 |
140 | SAVE_VSX 24, 352, 9 |
141 | SAVE_VSX 25, 368, 9 |
142 | SAVE_VSX 26, 384, 9 |
143 | SAVE_VSX 27, 400, 9 |
144 | SAVE_VSX 28, 416, 9 |
145 | SAVE_VSX 29, 432, 9 |
146 | SAVE_VSX 30, 448, 9 |
147 | SAVE_VSX 31, 464, 9 |
148 | .endm # SAVE_REGS |
149 | |
150 | .macro RESTORE_REGS |
151 | addi 9, 1, 256 |
152 | RESTORE_VRS 20, 0, 9 |
153 | RESTORE_VRS 21, 16, 9 |
154 | RESTORE_VRS 22, 32, 9 |
155 | RESTORE_VRS 23, 48, 9 |
156 | RESTORE_VRS 24, 64, 9 |
157 | RESTORE_VRS 25, 80, 9 |
158 | RESTORE_VRS 26, 96, 9 |
159 | RESTORE_VRS 27, 112, 9 |
160 | RESTORE_VRS 28, 128, 9 |
161 | RESTORE_VRS 29, 144, 9 |
162 | RESTORE_VRS 30, 160, 9 |
163 | RESTORE_VRS 31, 176, 9 |
164 | |
165 | RESTORE_VSX 14, 192, 9 |
166 | RESTORE_VSX 15, 208, 9 |
167 | RESTORE_VSX 16, 224, 9 |
168 | RESTORE_VSX 17, 240, 9 |
169 | RESTORE_VSX 18, 256, 9 |
170 | RESTORE_VSX 19, 272, 9 |
171 | RESTORE_VSX 20, 288, 9 |
172 | RESTORE_VSX 21, 304, 9 |
173 | RESTORE_VSX 22, 320, 9 |
174 | RESTORE_VSX 23, 336, 9 |
175 | RESTORE_VSX 24, 352, 9 |
176 | RESTORE_VSX 25, 368, 9 |
177 | RESTORE_VSX 26, 384, 9 |
178 | RESTORE_VSX 27, 400, 9 |
179 | RESTORE_VSX 28, 416, 9 |
180 | RESTORE_VSX 29, 432, 9 |
181 | RESTORE_VSX 30, 448, 9 |
182 | RESTORE_VSX 31, 464, 9 |
183 | |
184 | RESTORE_GPR 14, 112, 1 |
185 | RESTORE_GPR 15, 120, 1 |
186 | RESTORE_GPR 16, 128, 1 |
187 | RESTORE_GPR 17, 136, 1 |
188 | RESTORE_GPR 18, 144, 1 |
189 | RESTORE_GPR 19, 152, 1 |
190 | RESTORE_GPR 20, 160, 1 |
191 | RESTORE_GPR 21, 168, 1 |
192 | RESTORE_GPR 22, 176, 1 |
193 | RESTORE_GPR 23, 184, 1 |
194 | RESTORE_GPR 24, 192, 1 |
195 | RESTORE_GPR 25, 200, 1 |
196 | RESTORE_GPR 26, 208, 1 |
197 | RESTORE_GPR 27, 216, 1 |
198 | RESTORE_GPR 28, 224, 1 |
199 | RESTORE_GPR 29, 232, 1 |
200 | RESTORE_GPR 30, 240, 1 |
201 | RESTORE_GPR 31, 248, 1 |
202 | |
203 | addi 1, 1, 752 |
204 | ld 0, 16(1) |
205 | mtlr 0 |
206 | .endm # RESTORE_REGS |
207 | |
208 | # |
209 | # p[0] = a0*r0 + a1*r4*5 + a2*r3*5 + a3*r2*5 + a4*r1*5; |
210 | # p[1] = a0*r1 + a1*r0 + a2*r4*5 + a3*r3*5 + a4*r2*5; |
211 | # p[2] = a0*r2 + a1*r1 + a2*r0 + a3*r4*5 + a4*r3*5; |
212 | # p[3] = a0*r3 + a1*r2 + a2*r1 + a3*r0 + a4*r4*5; |
213 | # p[4] = a0*r4 + a1*r3 + a2*r2 + a3*r1 + a4*r0 ; |
214 | # |
215 | # [r^2, r^3, r^1, r^4] |
216 | # [m3, m2, m4, m1] |
217 | # |
218 | # multiply odd and even words |
219 | .macro mul_odd |
220 | vmulouw 14, 4, 26 |
221 | vmulouw 10, 5, 3 |
222 | vmulouw 11, 6, 2 |
223 | vmulouw 12, 7, 1 |
224 | vmulouw 13, 8, 0 |
225 | vmulouw 15, 4, 27 |
226 | vaddudm 14, 14, 10 |
227 | vaddudm 14, 14, 11 |
228 | vmulouw 10, 5, 26 |
229 | vmulouw 11, 6, 3 |
230 | vaddudm 14, 14, 12 |
231 | vaddudm 14, 14, 13 # x0 |
232 | vaddudm 15, 15, 10 |
233 | vaddudm 15, 15, 11 |
234 | vmulouw 12, 7, 2 |
235 | vmulouw 13, 8, 1 |
236 | vaddudm 15, 15, 12 |
237 | vaddudm 15, 15, 13 # x1 |
238 | vmulouw 16, 4, 28 |
239 | vmulouw 10, 5, 27 |
240 | vmulouw 11, 6, 26 |
241 | vaddudm 16, 16, 10 |
242 | vaddudm 16, 16, 11 |
243 | vmulouw 12, 7, 3 |
244 | vmulouw 13, 8, 2 |
245 | vaddudm 16, 16, 12 |
246 | vaddudm 16, 16, 13 # x2 |
247 | vmulouw 17, 4, 29 |
248 | vmulouw 10, 5, 28 |
249 | vmulouw 11, 6, 27 |
250 | vaddudm 17, 17, 10 |
251 | vaddudm 17, 17, 11 |
252 | vmulouw 12, 7, 26 |
253 | vmulouw 13, 8, 3 |
254 | vaddudm 17, 17, 12 |
255 | vaddudm 17, 17, 13 # x3 |
256 | vmulouw 18, 4, 30 |
257 | vmulouw 10, 5, 29 |
258 | vmulouw 11, 6, 28 |
259 | vaddudm 18, 18, 10 |
260 | vaddudm 18, 18, 11 |
261 | vmulouw 12, 7, 27 |
262 | vmulouw 13, 8, 26 |
263 | vaddudm 18, 18, 12 |
264 | vaddudm 18, 18, 13 # x4 |
265 | .endm |
266 | |
267 | .macro mul_even |
268 | vmuleuw 9, 4, 26 |
269 | vmuleuw 10, 5, 3 |
270 | vmuleuw 11, 6, 2 |
271 | vmuleuw 12, 7, 1 |
272 | vmuleuw 13, 8, 0 |
273 | vaddudm 14, 14, 9 |
274 | vaddudm 14, 14, 10 |
275 | vaddudm 14, 14, 11 |
276 | vaddudm 14, 14, 12 |
277 | vaddudm 14, 14, 13 # x0 |
278 | |
279 | vmuleuw 9, 4, 27 |
280 | vmuleuw 10, 5, 26 |
281 | vmuleuw 11, 6, 3 |
282 | vmuleuw 12, 7, 2 |
283 | vmuleuw 13, 8, 1 |
284 | vaddudm 15, 15, 9 |
285 | vaddudm 15, 15, 10 |
286 | vaddudm 15, 15, 11 |
287 | vaddudm 15, 15, 12 |
288 | vaddudm 15, 15, 13 # x1 |
289 | |
290 | vmuleuw 9, 4, 28 |
291 | vmuleuw 10, 5, 27 |
292 | vmuleuw 11, 6, 26 |
293 | vmuleuw 12, 7, 3 |
294 | vmuleuw 13, 8, 2 |
295 | vaddudm 16, 16, 9 |
296 | vaddudm 16, 16, 10 |
297 | vaddudm 16, 16, 11 |
298 | vaddudm 16, 16, 12 |
299 | vaddudm 16, 16, 13 # x2 |
300 | |
301 | vmuleuw 9, 4, 29 |
302 | vmuleuw 10, 5, 28 |
303 | vmuleuw 11, 6, 27 |
304 | vmuleuw 12, 7, 26 |
305 | vmuleuw 13, 8, 3 |
306 | vaddudm 17, 17, 9 |
307 | vaddudm 17, 17, 10 |
308 | vaddudm 17, 17, 11 |
309 | vaddudm 17, 17, 12 |
310 | vaddudm 17, 17, 13 # x3 |
311 | |
312 | vmuleuw 9, 4, 30 |
313 | vmuleuw 10, 5, 29 |
314 | vmuleuw 11, 6, 28 |
315 | vmuleuw 12, 7, 27 |
316 | vmuleuw 13, 8, 26 |
317 | vaddudm 18, 18, 9 |
318 | vaddudm 18, 18, 10 |
319 | vaddudm 18, 18, 11 |
320 | vaddudm 18, 18, 12 |
321 | vaddudm 18, 18, 13 # x4 |
322 | .endm |
323 | |
324 | # |
325 | # poly1305_setup_r |
326 | # |
327 | # setup r^4, r^3, r^2, r vectors |
328 | # [r, r^3, r^2, r^4] |
329 | # vs0 = [r0,...] |
330 | # vs1 = [r1,...] |
331 | # vs2 = [r2,...] |
332 | # vs3 = [r3,...] |
333 | # vs4 = [r4,...] |
334 | # vs5 = [r4*5,...] |
335 | # vs6 = [r3*5,...] |
336 | # vs7 = [r2*5,...] |
337 | # vs8 = [r1*5,...] |
338 | # |
339 | # r0, r4*5, r3*5, r2*5, r1*5; |
340 | # r1, r0, r4*5, r3*5, r2*5; |
341 | # r2, r1, r0, r4*5, r3*5; |
342 | # r3, r2, r1, r0, r4*5; |
343 | # r4, r3, r2, r1, r0 ; |
344 | # |
345 | .macro poly1305_setup_r |
346 | |
347 | # save r |
348 | xxlor 26, 58, 58 |
349 | xxlor 27, 59, 59 |
350 | xxlor 28, 60, 60 |
351 | xxlor 29, 61, 61 |
352 | xxlor 30, 62, 62 |
353 | |
354 | xxlxor 31, 31, 31 |
355 | |
356 | # [r, r^3, r^2, r^4] |
357 | # compute r^2 |
358 | vmr 4, 26 |
359 | vmr 5, 27 |
360 | vmr 6, 28 |
361 | vmr 7, 29 |
362 | vmr 8, 30 |
363 | bl do_mul # r^2 r^1 |
364 | xxpermdi 58, 58, 36, 0x3 # r0 |
365 | xxpermdi 59, 59, 37, 0x3 # r1 |
366 | xxpermdi 60, 60, 38, 0x3 # r2 |
367 | xxpermdi 61, 61, 39, 0x3 # r3 |
368 | xxpermdi 62, 62, 40, 0x3 # r4 |
369 | xxpermdi 36, 36, 36, 0x3 |
370 | xxpermdi 37, 37, 37, 0x3 |
371 | xxpermdi 38, 38, 38, 0x3 |
372 | xxpermdi 39, 39, 39, 0x3 |
373 | xxpermdi 40, 40, 40, 0x3 |
374 | vspltisb 13, 2 |
375 | vsld 9, 27, 13 |
376 | vsld 10, 28, 13 |
377 | vsld 11, 29, 13 |
378 | vsld 12, 30, 13 |
379 | vaddudm 0, 9, 27 |
380 | vaddudm 1, 10, 28 |
381 | vaddudm 2, 11, 29 |
382 | vaddudm 3, 12, 30 |
383 | |
384 | bl do_mul # r^4 r^3 |
385 | vmrgow 26, 26, 4 |
386 | vmrgow 27, 27, 5 |
387 | vmrgow 28, 28, 6 |
388 | vmrgow 29, 29, 7 |
389 | vmrgow 30, 30, 8 |
390 | vspltisb 13, 2 |
391 | vsld 9, 27, 13 |
392 | vsld 10, 28, 13 |
393 | vsld 11, 29, 13 |
394 | vsld 12, 30, 13 |
395 | vaddudm 0, 9, 27 |
396 | vaddudm 1, 10, 28 |
397 | vaddudm 2, 11, 29 |
398 | vaddudm 3, 12, 30 |
399 | |
400 | # r^2 r^4 |
401 | xxlor 0, 58, 58 |
402 | xxlor 1, 59, 59 |
403 | xxlor 2, 60, 60 |
404 | xxlor 3, 61, 61 |
405 | xxlor 4, 62, 62 |
406 | xxlor 5, 32, 32 |
407 | xxlor 6, 33, 33 |
408 | xxlor 7, 34, 34 |
409 | xxlor 8, 35, 35 |
410 | |
411 | vspltw 9, 26, 3 |
412 | vspltw 10, 26, 2 |
413 | vmrgow 26, 10, 9 |
414 | vspltw 9, 27, 3 |
415 | vspltw 10, 27, 2 |
416 | vmrgow 27, 10, 9 |
417 | vspltw 9, 28, 3 |
418 | vspltw 10, 28, 2 |
419 | vmrgow 28, 10, 9 |
420 | vspltw 9, 29, 3 |
421 | vspltw 10, 29, 2 |
422 | vmrgow 29, 10, 9 |
423 | vspltw 9, 30, 3 |
424 | vspltw 10, 30, 2 |
425 | vmrgow 30, 10, 9 |
426 | |
427 | vsld 9, 27, 13 |
428 | vsld 10, 28, 13 |
429 | vsld 11, 29, 13 |
430 | vsld 12, 30, 13 |
431 | vaddudm 0, 9, 27 |
432 | vaddudm 1, 10, 28 |
433 | vaddudm 2, 11, 29 |
434 | vaddudm 3, 12, 30 |
435 | .endm |
436 | |
437 | SYM_FUNC_START_LOCAL(do_mul) |
438 | mul_odd |
439 | |
440 | # do reduction ( h %= p ) |
441 | # carry reduction |
442 | vspltisb 9, 2 |
443 | vsrd 10, 14, 31 |
444 | vsrd 11, 17, 31 |
445 | vand 7, 17, 25 |
446 | vand 4, 14, 25 |
447 | vaddudm 18, 18, 11 |
448 | vsrd 12, 18, 31 |
449 | vaddudm 15, 15, 10 |
450 | |
451 | vsrd 11, 15, 31 |
452 | vand 8, 18, 25 |
453 | vand 5, 15, 25 |
454 | vaddudm 4, 4, 12 |
455 | vsld 10, 12, 9 |
456 | vaddudm 6, 16, 11 |
457 | |
458 | vsrd 13, 6, 31 |
459 | vand 6, 6, 25 |
460 | vaddudm 4, 4, 10 |
461 | vsrd 10, 4, 31 |
462 | vaddudm 7, 7, 13 |
463 | |
464 | vsrd 11, 7, 31 |
465 | vand 7, 7, 25 |
466 | vand 4, 4, 25 |
467 | vaddudm 5, 5, 10 |
468 | vaddudm 8, 8, 11 |
469 | blr |
470 | SYM_FUNC_END(do_mul) |
471 | |
472 | # |
473 | # init key |
474 | # |
475 | .macro do_poly1305_init |
476 | addis 10, 2, rmask@toc@ha |
477 | addi 10, 10, rmask@toc@l |
478 | |
479 | ld 11, 0(10) |
480 | ld 12, 8(10) |
481 | |
482 | li 14, 16 |
483 | li 15, 32 |
484 | addis 10, 2, cnum@toc@ha |
485 | addi 10, 10, cnum@toc@l |
486 | lvx 25, 0, 10 # v25 - mask |
487 | lvx 31, 14, 10 # v31 = 1a |
488 | lvx 19, 15, 10 # v19 = 1 << 24 |
489 | lxv 24, 48(10) # vs24 |
490 | lxv 25, 64(10) # vs25 |
491 | |
492 | # initialize |
493 | # load key from r3 to vectors |
494 | ld 9, 24(3) |
495 | ld 10, 32(3) |
496 | and. 9, 9, 11 |
497 | and. 10, 10, 12 |
498 | |
499 | # break 26 bits |
500 | extrdi 14, 9, 26, 38 |
501 | extrdi 15, 9, 26, 12 |
502 | extrdi 16, 9, 12, 0 |
503 | mtvsrdd 58, 0, 14 |
504 | insrdi 16, 10, 14, 38 |
505 | mtvsrdd 59, 0, 15 |
506 | extrdi 17, 10, 26, 24 |
507 | mtvsrdd 60, 0, 16 |
508 | extrdi 18, 10, 24, 0 |
509 | mtvsrdd 61, 0, 17 |
510 | mtvsrdd 62, 0, 18 |
511 | |
512 | # r1 = r1 * 5, r2 = r2 * 5, r3 = r3 * 5, r4 = r4 * 5 |
513 | li 9, 5 |
514 | mtvsrdd 36, 0, 9 |
515 | vmulouw 0, 27, 4 # v0 = rr0 |
516 | vmulouw 1, 28, 4 # v1 = rr1 |
517 | vmulouw 2, 29, 4 # v2 = rr2 |
518 | vmulouw 3, 30, 4 # v3 = rr3 |
519 | .endm |
520 | |
521 | # |
522 | # poly1305_p10le_4blocks( uint8_t *k, uint32_t mlen, uint8_t *m) |
523 | # k = 32 bytes key |
524 | # r3 = k (r, s) |
525 | # r4 = mlen |
526 | # r5 = m |
527 | # |
528 | SYM_FUNC_START(poly1305_p10le_4blocks) |
529 | .align 5 |
530 | cmpdi 5, 64 |
531 | blt Out_no_poly1305 |
532 | |
533 | SAVE_REGS |
534 | |
535 | do_poly1305_init |
536 | |
537 | li 21, 0 # counter to message |
538 | |
539 | poly1305_setup_r |
540 | |
541 | # load previous H state |
542 | # break/convert r6 to 26 bits |
543 | ld 9, 0(3) |
544 | ld 10, 8(3) |
545 | ld 19, 16(3) |
546 | sldi 19, 19, 24 |
547 | mtvsrdd 41, 0, 19 |
548 | extrdi 14, 9, 26, 38 |
549 | extrdi 15, 9, 26, 12 |
550 | extrdi 16, 9, 12, 0 |
551 | mtvsrdd 36, 0, 14 |
552 | insrdi 16, 10, 14, 38 |
553 | mtvsrdd 37, 0, 15 |
554 | extrdi 17, 10, 26, 24 |
555 | mtvsrdd 38, 0, 16 |
556 | extrdi 18, 10, 24, 0 |
557 | mtvsrdd 39, 0, 17 |
558 | mtvsrdd 40, 0, 18 |
559 | vor 8, 8, 9 |
560 | |
561 | # input m1 m2 |
562 | add 20, 4, 21 |
563 | xxlor 49, 24, 24 |
564 | xxlor 50, 25, 25 |
565 | lxvw4x 43, 0, 20 |
566 | addi 17, 20, 16 |
567 | lxvw4x 44, 0, 17 |
568 | vperm 14, 11, 12, 17 |
569 | vperm 15, 11, 12, 18 |
570 | vand 9, 14, 25 # a0 |
571 | vsrd 10, 14, 31 # >> 26 |
572 | vsrd 11, 10, 31 # 12 bits left |
573 | vand 10, 10, 25 # a1 |
574 | vspltisb 13, 12 |
575 | vand 16, 15, 25 |
576 | vsld 12, 16, 13 |
577 | vor 11, 11, 12 |
578 | vand 11, 11, 25 # a2 |
579 | vspltisb 13, 14 |
580 | vsrd 12, 15, 13 # >> 14 |
581 | vsrd 13, 12, 31 # >> 26, a4 |
582 | vand 12, 12, 25 # a3 |
583 | |
584 | vaddudm 20, 4, 9 |
585 | vaddudm 21, 5, 10 |
586 | vaddudm 22, 6, 11 |
587 | vaddudm 23, 7, 12 |
588 | vaddudm 24, 8, 13 |
589 | |
590 | # m3 m4 |
591 | addi 17, 17, 16 |
592 | lxvw4x 43, 0, 17 |
593 | addi 17, 17, 16 |
594 | lxvw4x 44, 0, 17 |
595 | vperm 14, 11, 12, 17 |
596 | vperm 15, 11, 12, 18 |
597 | vand 9, 14, 25 # a0 |
598 | vsrd 10, 14, 31 # >> 26 |
599 | vsrd 11, 10, 31 # 12 bits left |
600 | vand 10, 10, 25 # a1 |
601 | vspltisb 13, 12 |
602 | vand 16, 15, 25 |
603 | vsld 12, 16, 13 |
604 | vspltisb 13, 14 |
605 | vor 11, 11, 12 |
606 | vand 11, 11, 25 # a2 |
607 | vsrd 12, 15, 13 # >> 14 |
608 | vsrd 13, 12, 31 # >> 26, a4 |
609 | vand 12, 12, 25 # a3 |
610 | |
611 | # Smash 4 message blocks into 5 vectors of [m4, m2, m3, m1] |
612 | vmrgow 4, 9, 20 |
613 | vmrgow 5, 10, 21 |
614 | vmrgow 6, 11, 22 |
615 | vmrgow 7, 12, 23 |
616 | vmrgow 8, 13, 24 |
617 | vaddudm 8, 8, 19 |
618 | |
619 | addi 5, 5, -64 # len -= 64 |
620 | addi 21, 21, 64 # offset += 64 |
621 | |
622 | li 9, 64 |
623 | divdu 31, 5, 9 |
624 | |
625 | cmpdi 31, 0 |
626 | ble Skip_block_loop |
627 | |
628 | mtctr 31 |
629 | |
630 | # h4 = m1 * r⁴ + m2 * r³ + m3 * r² + m4 * r |
631 | # Rewrite the polynominal sum of product as follows, |
632 | # h1 = (h0 + m1) * r^2, h2 = (h0 + m2) * r^2 |
633 | # h3 = (h1 + m3) * r^2, h4 = (h2 + m4) * r^2 --> (h0 + m1) r*4 + (h3 + m3) r^2, (h0 + m2) r^4 + (h0 + m4) r^2 |
634 | # .... Repeat |
635 | # h5 = (h3 + m5) * r^2, h6 = (h4 + m6) * r^2 --> |
636 | # h7 = (h5 + m7) * r^2, h8 = (h6 + m8) * r^1 --> m5 * r^4 + m6 * r^3 + m7 * r^2 + m8 * r |
637 | # |
638 | loop_4blocks: |
639 | |
640 | # Multiply odd words and even words |
641 | mul_odd |
642 | mul_even |
643 | # carry reduction |
644 | vspltisb 9, 2 |
645 | vsrd 10, 14, 31 |
646 | vsrd 11, 17, 31 |
647 | vand 7, 17, 25 |
648 | vand 4, 14, 25 |
649 | vaddudm 18, 18, 11 |
650 | vsrd 12, 18, 31 |
651 | vaddudm 15, 15, 10 |
652 | |
653 | vsrd 11, 15, 31 |
654 | vand 8, 18, 25 |
655 | vand 5, 15, 25 |
656 | vaddudm 4, 4, 12 |
657 | vsld 10, 12, 9 |
658 | vaddudm 6, 16, 11 |
659 | |
660 | vsrd 13, 6, 31 |
661 | vand 6, 6, 25 |
662 | vaddudm 4, 4, 10 |
663 | vsrd 10, 4, 31 |
664 | vaddudm 7, 7, 13 |
665 | |
666 | vsrd 11, 7, 31 |
667 | vand 7, 7, 25 |
668 | vand 4, 4, 25 |
669 | vaddudm 5, 5, 10 |
670 | vaddudm 8, 8, 11 |
671 | |
672 | # input m1 m2 m3 m4 |
673 | add 20, 4, 21 |
674 | xxlor 49, 24, 24 |
675 | xxlor 50, 25, 25 |
676 | lxvw4x 43, 0, 20 |
677 | addi 17, 20, 16 |
678 | lxvw4x 44, 0, 17 |
679 | vperm 14, 11, 12, 17 |
680 | vperm 15, 11, 12, 18 |
681 | addi 17, 17, 16 |
682 | lxvw4x 43, 0, 17 |
683 | addi 17, 17, 16 |
684 | lxvw4x 44, 0, 17 |
685 | vperm 17, 11, 12, 17 |
686 | vperm 18, 11, 12, 18 |
687 | |
688 | vand 20, 14, 25 # a0 |
689 | vand 9, 17, 25 # a0 |
690 | vsrd 21, 14, 31 # >> 26 |
691 | vsrd 22, 21, 31 # 12 bits left |
692 | vsrd 10, 17, 31 # >> 26 |
693 | vsrd 11, 10, 31 # 12 bits left |
694 | |
695 | vand 21, 21, 25 # a1 |
696 | vand 10, 10, 25 # a1 |
697 | |
698 | vspltisb 13, 12 |
699 | vand 16, 15, 25 |
700 | vsld 23, 16, 13 |
701 | vor 22, 22, 23 |
702 | vand 22, 22, 25 # a2 |
703 | vand 16, 18, 25 |
704 | vsld 12, 16, 13 |
705 | vor 11, 11, 12 |
706 | vand 11, 11, 25 # a2 |
707 | vspltisb 13, 14 |
708 | vsrd 23, 15, 13 # >> 14 |
709 | vsrd 24, 23, 31 # >> 26, a4 |
710 | vand 23, 23, 25 # a3 |
711 | vsrd 12, 18, 13 # >> 14 |
712 | vsrd 13, 12, 31 # >> 26, a4 |
713 | vand 12, 12, 25 # a3 |
714 | |
715 | vaddudm 4, 4, 20 |
716 | vaddudm 5, 5, 21 |
717 | vaddudm 6, 6, 22 |
718 | vaddudm 7, 7, 23 |
719 | vaddudm 8, 8, 24 |
720 | |
721 | # Smash 4 message blocks into 5 vectors of [m4, m2, m3, m1] |
722 | vmrgow 4, 9, 4 |
723 | vmrgow 5, 10, 5 |
724 | vmrgow 6, 11, 6 |
725 | vmrgow 7, 12, 7 |
726 | vmrgow 8, 13, 8 |
727 | vaddudm 8, 8, 19 |
728 | |
729 | addi 5, 5, -64 # len -= 64 |
730 | addi 21, 21, 64 # offset += 64 |
731 | |
732 | bdnz loop_4blocks |
733 | |
734 | Skip_block_loop: |
735 | xxlor 58, 0, 0 |
736 | xxlor 59, 1, 1 |
737 | xxlor 60, 2, 2 |
738 | xxlor 61, 3, 3 |
739 | xxlor 62, 4, 4 |
740 | xxlor 32, 5, 5 |
741 | xxlor 33, 6, 6 |
742 | xxlor 34, 7, 7 |
743 | xxlor 35, 8, 8 |
744 | |
745 | # Multiply odd words and even words |
746 | mul_odd |
747 | mul_even |
748 | |
749 | # Sum the products. |
750 | xxpermdi 41, 31, 46, 0 |
751 | xxpermdi 42, 31, 47, 0 |
752 | vaddudm 4, 14, 9 |
753 | xxpermdi 36, 31, 36, 3 |
754 | vaddudm 5, 15, 10 |
755 | xxpermdi 37, 31, 37, 3 |
756 | xxpermdi 43, 31, 48, 0 |
757 | vaddudm 6, 16, 11 |
758 | xxpermdi 38, 31, 38, 3 |
759 | xxpermdi 44, 31, 49, 0 |
760 | vaddudm 7, 17, 12 |
761 | xxpermdi 39, 31, 39, 3 |
762 | xxpermdi 45, 31, 50, 0 |
763 | vaddudm 8, 18, 13 |
764 | xxpermdi 40, 31, 40, 3 |
765 | |
766 | # carry reduction |
767 | vspltisb 9, 2 |
768 | vsrd 10, 4, 31 |
769 | vsrd 11, 7, 31 |
770 | vand 7, 7, 25 |
771 | vand 4, 4, 25 |
772 | vaddudm 8, 8, 11 |
773 | vsrd 12, 8, 31 |
774 | vaddudm 5, 5, 10 |
775 | |
776 | vsrd 11, 5, 31 |
777 | vand 8, 8, 25 |
778 | vand 5, 5, 25 |
779 | vaddudm 4, 4, 12 |
780 | vsld 10, 12, 9 |
781 | vaddudm 6, 6, 11 |
782 | |
783 | vsrd 13, 6, 31 |
784 | vand 6, 6, 25 |
785 | vaddudm 4, 4, 10 |
786 | vsrd 10, 4, 31 |
787 | vaddudm 7, 7, 13 |
788 | |
789 | vsrd 11, 7, 31 |
790 | vand 7, 7, 25 |
791 | vand 4, 4, 25 |
792 | vaddudm 5, 5, 10 |
793 | vsrd 10, 5, 31 |
794 | vand 5, 5, 25 |
795 | vaddudm 6, 6, 10 |
796 | vaddudm 8, 8, 11 |
797 | |
798 | b do_final_update |
799 | |
800 | do_final_update: |
801 | # combine 26 bit limbs |
802 | # v4, v5, v6, v7 and v8 are 26 bit vectors |
803 | vsld 5, 5, 31 |
804 | vor 20, 4, 5 |
805 | vspltisb 11, 12 |
806 | vsrd 12, 6, 11 |
807 | vsld 6, 6, 31 |
808 | vsld 6, 6, 31 |
809 | vor 20, 20, 6 |
810 | vspltisb 11, 14 |
811 | vsld 7, 7, 11 |
812 | vor 21, 7, 12 |
813 | mfvsrld 16, 40 # save last 2 bytes |
814 | vsld 8, 8, 11 |
815 | vsld 8, 8, 31 |
816 | vor 21, 21, 8 |
817 | mfvsrld 17, 52 |
818 | mfvsrld 19, 53 |
819 | srdi 16, 16, 24 |
820 | |
821 | std 17, 0(3) |
822 | std 19, 8(3) |
823 | stw 16, 16(3) |
824 | |
825 | Out_loop: |
826 | li 3, 0 |
827 | |
828 | RESTORE_REGS |
829 | |
830 | blr |
831 | |
832 | Out_no_poly1305: |
833 | li 3, 0 |
834 | blr |
835 | SYM_FUNC_END(poly1305_p10le_4blocks) |
836 | |
837 | # |
838 | # ======================================================================= |
839 | # The following functions implement 64 x 64 bits multiplication poly1305. |
840 | # |
841 | SYM_FUNC_START_LOCAL(Poly1305_init_64) |
842 | # mask 0x0FFFFFFC0FFFFFFC |
843 | # mask 0x0FFFFFFC0FFFFFFF |
844 | addis 10, 2, rmask@toc@ha |
845 | addi 10, 10, rmask@toc@l |
846 | ld 11, 0(10) |
847 | ld 12, 8(10) |
848 | |
849 | # initialize |
850 | # load key from r3 |
851 | ld 9, 24(3) |
852 | ld 10, 32(3) |
853 | and. 9, 9, 11 # cramp mask r0 |
854 | and. 10, 10, 12 # cramp mask r1 |
855 | |
856 | srdi 21, 10, 2 |
857 | add 19, 21, 10 # s1: r19 - (r1 >> 2) *5 |
858 | |
859 | # setup r and s |
860 | li 25, 0 |
861 | mtvsrdd 32+0, 9, 19 # r0, s1 |
862 | mtvsrdd 32+1, 10, 9 # r1, r0 |
863 | mtvsrdd 32+2, 19, 25 # s1 |
864 | mtvsrdd 32+3, 9, 25 # r0 |
865 | |
866 | blr |
867 | SYM_FUNC_END(Poly1305_init_64) |
868 | |
869 | # Poly1305_mult |
870 | # v6 = (h0, h1), v8 = h2 |
871 | # v0 = (r0, s1), v1 = (r1, r0), v2 = s1, v3 = r0 |
872 | # |
873 | # Output: v7, v10, v11 |
874 | # |
875 | SYM_FUNC_START_LOCAL(Poly1305_mult) |
876 | # |
877 | # d0 = h0 * r0 + h1 * s1 |
878 | vmsumudm 7, 6, 0, 9 # h0 * r0, h1 * s1 |
879 | |
880 | # d1 = h0 * r1 + h1 * r0 + h2 * s1 |
881 | vmsumudm 11, 6, 1, 9 # h0 * r1, h1 * r0 |
882 | vmsumudm 10, 8, 2, 11 # d1 += h2 * s1 |
883 | |
884 | # d2 = r0 |
885 | vmsumudm 11, 8, 3, 9 # d2 = h2 * r0 |
886 | blr |
887 | SYM_FUNC_END(Poly1305_mult) |
888 | |
889 | # |
890 | # carry reduction |
891 | # h %=p |
892 | # |
893 | # Input: v7, v10, v11 |
894 | # Output: r27, r28, r29 |
895 | # |
896 | SYM_FUNC_START_LOCAL(Carry_reduction) |
897 | mfvsrld 27, 32+7 |
898 | mfvsrld 28, 32+10 |
899 | mfvsrld 29, 32+11 |
900 | mfvsrd 20, 32+7 # h0.h |
901 | mfvsrd 21, 32+10 # h1.h |
902 | |
903 | addc 28, 28, 20 |
904 | adde 29, 29, 21 |
905 | srdi 22, 29, 0x2 |
906 | sldi 23, 22, 0x2 |
907 | add 23, 23, 22 # (h2 & 3) * 5 |
908 | addc 27, 27, 23 # h0 |
909 | addze 28, 28 # h1 |
910 | andi. 29, 29, 0x3 # h2 |
911 | blr |
912 | SYM_FUNC_END(Carry_reduction) |
913 | |
914 | # |
915 | # poly1305 multiplication |
916 | # h *= r, h %= p |
917 | # d0 = h0 * r0 + h1 * s1 |
918 | # d1 = h0 * r1 + h1 * r0 + h2 * s1 |
919 | # d2 = h0 * r0 |
920 | # |
921 | # |
922 | # unsigned int poly1305_test_64s(unisgned char *state, const byte *src, size_t len, highbit) |
923 | # - no highbit if final leftover block (highbit = 0) |
924 | # |
925 | SYM_FUNC_START(poly1305_64s) |
926 | cmpdi 5, 0 |
927 | ble Out_no_poly1305_64 |
928 | |
929 | mflr 0 |
930 | std 0, 16(1) |
931 | stdu 1,-400(1) |
932 | |
933 | SAVE_GPR 14, 112, 1 |
934 | SAVE_GPR 15, 120, 1 |
935 | SAVE_GPR 16, 128, 1 |
936 | SAVE_GPR 17, 136, 1 |
937 | SAVE_GPR 18, 144, 1 |
938 | SAVE_GPR 19, 152, 1 |
939 | SAVE_GPR 20, 160, 1 |
940 | SAVE_GPR 21, 168, 1 |
941 | SAVE_GPR 22, 176, 1 |
942 | SAVE_GPR 23, 184, 1 |
943 | SAVE_GPR 24, 192, 1 |
944 | SAVE_GPR 25, 200, 1 |
945 | SAVE_GPR 26, 208, 1 |
946 | SAVE_GPR 27, 216, 1 |
947 | SAVE_GPR 28, 224, 1 |
948 | SAVE_GPR 29, 232, 1 |
949 | SAVE_GPR 30, 240, 1 |
950 | SAVE_GPR 31, 248, 1 |
951 | |
952 | # Init poly1305 |
953 | bl Poly1305_init_64 |
954 | |
955 | li 25, 0 # offset to inp and outp |
956 | |
957 | add 11, 25, 4 |
958 | |
959 | # load h |
960 | # h0, h1, h2? |
961 | ld 27, 0(3) |
962 | ld 28, 8(3) |
963 | lwz 29, 16(3) |
964 | |
965 | li 30, 16 |
966 | divdu 31, 5, 30 |
967 | |
968 | mtctr 31 |
969 | |
970 | mr 24, 6 # highbit |
971 | |
972 | Loop_block_64: |
973 | vxor 9, 9, 9 |
974 | |
975 | ld 20, 0(11) |
976 | ld 21, 8(11) |
977 | addi 11, 11, 16 |
978 | |
979 | addc 27, 27, 20 |
980 | adde 28, 28, 21 |
981 | adde 29, 29, 24 |
982 | |
983 | li 22, 0 |
984 | mtvsrdd 32+6, 27, 28 # h0, h1 |
985 | mtvsrdd 32+8, 29, 22 # h2 |
986 | |
987 | bl Poly1305_mult |
988 | |
989 | bl Carry_reduction |
990 | |
991 | bdnz Loop_block_64 |
992 | |
993 | std 27, 0(3) |
994 | std 28, 8(3) |
995 | stw 29, 16(3) |
996 | |
997 | li 3, 0 |
998 | |
999 | RESTORE_GPR 14, 112, 1 |
1000 | RESTORE_GPR 15, 120, 1 |
1001 | RESTORE_GPR 16, 128, 1 |
1002 | RESTORE_GPR 17, 136, 1 |
1003 | RESTORE_GPR 18, 144, 1 |
1004 | RESTORE_GPR 19, 152, 1 |
1005 | RESTORE_GPR 20, 160, 1 |
1006 | RESTORE_GPR 21, 168, 1 |
1007 | RESTORE_GPR 22, 176, 1 |
1008 | RESTORE_GPR 23, 184, 1 |
1009 | RESTORE_GPR 24, 192, 1 |
1010 | RESTORE_GPR 25, 200, 1 |
1011 | RESTORE_GPR 26, 208, 1 |
1012 | RESTORE_GPR 27, 216, 1 |
1013 | RESTORE_GPR 28, 224, 1 |
1014 | RESTORE_GPR 29, 232, 1 |
1015 | RESTORE_GPR 30, 240, 1 |
1016 | RESTORE_GPR 31, 248, 1 |
1017 | |
1018 | addi 1, 1, 400 |
1019 | ld 0, 16(1) |
1020 | mtlr 0 |
1021 | |
1022 | blr |
1023 | |
1024 | Out_no_poly1305_64: |
1025 | li 3, 0 |
1026 | blr |
1027 | SYM_FUNC_END(poly1305_64s) |
1028 | |
1029 | # |
1030 | # Input: r3 = h, r4 = s, r5 = mac |
1031 | # mac = h + s |
1032 | # |
1033 | SYM_FUNC_START(poly1305_emit_64) |
1034 | ld 10, 0(3) |
1035 | ld 11, 8(3) |
1036 | ld 12, 16(3) |
1037 | |
1038 | # compare modulus |
1039 | # h + 5 + (-p) |
1040 | mr 6, 10 |
1041 | mr 7, 11 |
1042 | mr 8, 12 |
1043 | addic. 6, 6, 5 |
1044 | addze 7, 7 |
1045 | addze 8, 8 |
1046 | srdi 9, 8, 2 # overflow? |
1047 | cmpdi 9, 0 |
1048 | beq Skip_h64 |
1049 | mr 10, 6 |
1050 | mr 11, 7 |
1051 | mr 12, 8 |
1052 | |
1053 | Skip_h64: |
1054 | ld 6, 0(4) |
1055 | ld 7, 8(4) |
1056 | addc 10, 10, 6 |
1057 | adde 11, 11, 7 |
1058 | addze 12, 12 |
1059 | |
1060 | std 10, 0(5) |
1061 | std 11, 8(5) |
1062 | blr |
1063 | SYM_FUNC_END(poly1305_emit_64) |
1064 | |
1065 | SYM_DATA_START_LOCAL(RMASK) |
1066 | .align 5 |
1067 | rmask: |
1068 | .byte 0xff, 0xff, 0xff, 0x0f, 0xfc, 0xff, 0xff, 0x0f, 0xfc, 0xff, 0xff, 0x0f, 0xfc, 0xff, 0xff, 0x0f |
1069 | cnum: |
1070 | .long 0x03ffffff, 0x00000000, 0x03ffffff, 0x00000000 |
1071 | .long 0x1a, 0x00, 0x1a, 0x00 |
1072 | .long 0x01000000, 0x01000000, 0x01000000, 0x01000000 |
1073 | .long 0x00010203, 0x04050607, 0x10111213, 0x14151617 |
1074 | .long 0x08090a0b, 0x0c0d0e0f, 0x18191a1b, 0x1c1d1e1f |
1075 | SYM_DATA_END(RMASK) |
1076 | |