1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
2 | /* |
3 | * Fast SHA-256 implementation for SPE instruction set (PPC) |
4 | * |
5 | * This code makes use of the SPE SIMD instruction set as defined in |
6 | * http://cache.freescale.com/files/32bit/doc/ref_manual/SPEPIM.pdf |
7 | * Implementation is based on optimization guide notes from |
8 | * http://cache.freescale.com/files/32bit/doc/app_note/AN2665.pdf |
9 | * |
10 | * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de> |
11 | */ |
12 | |
13 | #include <asm/ppc_asm.h> |
14 | #include <asm/asm-offsets.h> |
15 | |
16 | #define rHP r3 /* pointer to hash values in memory */ |
17 | #define rKP r24 /* pointer to round constants */ |
18 | #define rWP r4 /* pointer to input data */ |
19 | |
20 | #define rH0 r5 /* 8 32 bit hash values in 8 registers */ |
21 | #define rH1 r6 |
22 | #define rH2 r7 |
23 | #define rH3 r8 |
24 | #define rH4 r9 |
25 | #define rH5 r10 |
26 | #define rH6 r11 |
27 | #define rH7 r12 |
28 | |
29 | #define rW0 r14 /* 64 bit registers. 16 words in 8 registers */ |
30 | #define rW1 r15 |
31 | #define rW2 r16 |
32 | #define rW3 r17 |
33 | #define rW4 r18 |
34 | #define rW5 r19 |
35 | #define rW6 r20 |
36 | #define rW7 r21 |
37 | |
38 | #define rT0 r22 /* 64 bit temporaries */ |
39 | #define rT1 r23 |
40 | #define rT2 r0 /* 32 bit temporaries */ |
41 | #define rT3 r25 |
42 | |
43 | #define CMP_KN_LOOP |
44 | #define CMP_KC_LOOP \ |
45 | cmpwi rT1,0; |
46 | |
47 | #define INITIALIZE \ |
48 | stwu r1,-128(r1); /* create stack frame */ \ |
49 | evstdw r14,8(r1); /* We must save non volatile */ \ |
50 | evstdw r15,16(r1); /* registers. Take the chance */ \ |
51 | evstdw r16,24(r1); /* and save the SPE part too */ \ |
52 | evstdw r17,32(r1); \ |
53 | evstdw r18,40(r1); \ |
54 | evstdw r19,48(r1); \ |
55 | evstdw r20,56(r1); \ |
56 | evstdw r21,64(r1); \ |
57 | evstdw r22,72(r1); \ |
58 | evstdw r23,80(r1); \ |
59 | stw r24,88(r1); /* save normal registers */ \ |
60 | stw r25,92(r1); |
61 | |
62 | |
63 | #define FINALIZE \ |
64 | evldw r14,8(r1); /* restore SPE registers */ \ |
65 | evldw r15,16(r1); \ |
66 | evldw r16,24(r1); \ |
67 | evldw r17,32(r1); \ |
68 | evldw r18,40(r1); \ |
69 | evldw r19,48(r1); \ |
70 | evldw r20,56(r1); \ |
71 | evldw r21,64(r1); \ |
72 | evldw r22,72(r1); \ |
73 | evldw r23,80(r1); \ |
74 | lwz r24,88(r1); /* restore normal registers */ \ |
75 | lwz r25,92(r1); \ |
76 | xor r0,r0,r0; \ |
77 | stw r0,8(r1); /* Delete sensitive data */ \ |
78 | stw r0,16(r1); /* that we might have pushed */ \ |
79 | stw r0,24(r1); /* from other context that runs */ \ |
80 | stw r0,32(r1); /* the same code. Assume that */ \ |
81 | stw r0,40(r1); /* the lower part of the GPRs */ \ |
82 | stw r0,48(r1); /* was already overwritten on */ \ |
83 | stw r0,56(r1); /* the way down to here */ \ |
84 | stw r0,64(r1); \ |
85 | stw r0,72(r1); \ |
86 | stw r0,80(r1); \ |
87 | addi r1,r1,128; /* cleanup stack frame */ |
88 | |
89 | #ifdef __BIG_ENDIAN__ |
90 | #define LOAD_DATA(reg, off) \ |
91 | lwz reg,off(rWP); /* load data */ |
92 | #define NEXT_BLOCK \ |
93 | addi rWP,rWP,64; /* increment per block */ |
94 | #else |
95 | #define LOAD_DATA(reg, off) \ |
96 | lwbrx reg,0,rWP; /* load data */ \ |
97 | addi rWP,rWP,4; /* increment per word */ |
98 | #define NEXT_BLOCK /* nothing to do */ |
99 | #endif |
100 | |
101 | #define R_LOAD_W(a, b, c, d, e, f, g, h, w, off) \ |
102 | LOAD_DATA(w, off) /* 1: W */ \ |
103 | rotrwi rT0,e,6; /* 1: S1 = e rotr 6 */ \ |
104 | rotrwi rT1,e,11; /* 1: S1' = e rotr 11 */ \ |
105 | rotrwi rT2,e,25; /* 1: S1" = e rotr 25 */ \ |
106 | xor rT0,rT0,rT1; /* 1: S1 = S1 xor S1' */ \ |
107 | and rT3,e,f; /* 1: ch = e and f */ \ |
108 | xor rT0,rT0,rT2; /* 1: S1 = S1 xor S1" */ \ |
109 | andc rT1,g,e; /* 1: ch' = ~e and g */ \ |
110 | lwz rT2,off(rKP); /* 1: K */ \ |
111 | xor rT3,rT3,rT1; /* 1: ch = ch xor ch' */ \ |
112 | add h,h,rT0; /* 1: temp1 = h + S1 */ \ |
113 | add rT3,rT3,w; /* 1: temp1' = ch + w */ \ |
114 | rotrwi rT0,a,2; /* 1: S0 = a rotr 2 */ \ |
115 | add h,h,rT3; /* 1: temp1 = temp1 + temp1' */ \ |
116 | rotrwi rT1,a,13; /* 1: S0' = a rotr 13 */ \ |
117 | add h,h,rT2; /* 1: temp1 = temp1 + K */ \ |
118 | rotrwi rT3,a,22; /* 1: S0" = a rotr 22 */ \ |
119 | xor rT0,rT0,rT1; /* 1: S0 = S0 xor S0' */ \ |
120 | add d,d,h; /* 1: d = d + temp1 */ \ |
121 | xor rT3,rT0,rT3; /* 1: S0 = S0 xor S0" */ \ |
122 | evmergelo w,w,w; /* shift W */ \ |
123 | or rT2,a,b; /* 1: maj = a or b */ \ |
124 | and rT1,a,b; /* 1: maj' = a and b */ \ |
125 | and rT2,rT2,c; /* 1: maj = maj and c */ \ |
126 | LOAD_DATA(w, off+4) /* 2: W */ \ |
127 | or rT2,rT1,rT2; /* 1: maj = maj or maj' */ \ |
128 | rotrwi rT0,d,6; /* 2: S1 = e rotr 6 */ \ |
129 | add rT3,rT3,rT2; /* 1: temp2 = S0 + maj */ \ |
130 | rotrwi rT1,d,11; /* 2: S1' = e rotr 11 */ \ |
131 | add h,h,rT3; /* 1: h = temp1 + temp2 */ \ |
132 | rotrwi rT2,d,25; /* 2: S1" = e rotr 25 */ \ |
133 | xor rT0,rT0,rT1; /* 2: S1 = S1 xor S1' */ \ |
134 | and rT3,d,e; /* 2: ch = e and f */ \ |
135 | xor rT0,rT0,rT2; /* 2: S1 = S1 xor S1" */ \ |
136 | andc rT1,f,d; /* 2: ch' = ~e and g */ \ |
137 | lwz rT2,off+4(rKP); /* 2: K */ \ |
138 | xor rT3,rT3,rT1; /* 2: ch = ch xor ch' */ \ |
139 | add g,g,rT0; /* 2: temp1 = h + S1 */ \ |
140 | add rT3,rT3,w; /* 2: temp1' = ch + w */ \ |
141 | rotrwi rT0,h,2; /* 2: S0 = a rotr 2 */ \ |
142 | add g,g,rT3; /* 2: temp1 = temp1 + temp1' */ \ |
143 | rotrwi rT1,h,13; /* 2: S0' = a rotr 13 */ \ |
144 | add g,g,rT2; /* 2: temp1 = temp1 + K */ \ |
145 | rotrwi rT3,h,22; /* 2: S0" = a rotr 22 */ \ |
146 | xor rT0,rT0,rT1; /* 2: S0 = S0 xor S0' */ \ |
147 | or rT2,h,a; /* 2: maj = a or b */ \ |
148 | xor rT3,rT0,rT3; /* 2: S0 = S0 xor S0" */ \ |
149 | and rT1,h,a; /* 2: maj' = a and b */ \ |
150 | and rT2,rT2,b; /* 2: maj = maj and c */ \ |
151 | add c,c,g; /* 2: d = d + temp1 */ \ |
152 | or rT2,rT1,rT2; /* 2: maj = maj or maj' */ \ |
153 | add rT3,rT3,rT2; /* 2: temp2 = S0 + maj */ \ |
154 | add g,g,rT3 /* 2: h = temp1 + temp2 */ |
155 | |
156 | #define R_CALC_W(a, b, c, d, e, f, g, h, w0, w1, w4, w5, w7, k, off) \ |
157 | rotrwi rT2,e,6; /* 1: S1 = e rotr 6 */ \ |
158 | evmergelohi rT0,w0,w1; /* w[-15] */ \ |
159 | rotrwi rT3,e,11; /* 1: S1' = e rotr 11 */ \ |
160 | evsrwiu rT1,rT0,3; /* s0 = w[-15] >> 3 */ \ |
161 | xor rT2,rT2,rT3; /* 1: S1 = S1 xor S1' */ \ |
162 | evrlwi rT0,rT0,25; /* s0' = w[-15] rotr 7 */ \ |
163 | rotrwi rT3,e,25; /* 1: S1' = e rotr 25 */ \ |
164 | evxor rT1,rT1,rT0; /* s0 = s0 xor s0' */ \ |
165 | xor rT2,rT2,rT3; /* 1: S1 = S1 xor S1' */ \ |
166 | evrlwi rT0,rT0,21; /* s0' = w[-15] rotr 18 */ \ |
167 | add h,h,rT2; /* 1: temp1 = h + S1 */ \ |
168 | evxor rT0,rT0,rT1; /* s0 = s0 xor s0' */ \ |
169 | and rT2,e,f; /* 1: ch = e and f */ \ |
170 | evaddw w0,w0,rT0; /* w = w[-16] + s0 */ \ |
171 | andc rT3,g,e; /* 1: ch' = ~e and g */ \ |
172 | evsrwiu rT0,w7,10; /* s1 = w[-2] >> 10 */ \ |
173 | xor rT2,rT2,rT3; /* 1: ch = ch xor ch' */ \ |
174 | evrlwi rT1,w7,15; /* s1' = w[-2] rotr 17 */ \ |
175 | add h,h,rT2; /* 1: temp1 = temp1 + ch */ \ |
176 | evxor rT0,rT0,rT1; /* s1 = s1 xor s1' */ \ |
177 | rotrwi rT2,a,2; /* 1: S0 = a rotr 2 */ \ |
178 | evrlwi rT1,w7,13; /* s1' = w[-2] rotr 19 */ \ |
179 | rotrwi rT3,a,13; /* 1: S0' = a rotr 13 */ \ |
180 | evxor rT0,rT0,rT1; /* s1 = s1 xor s1' */ \ |
181 | xor rT2,rT2,rT3; /* 1: S0 = S0 xor S0' */ \ |
182 | evldw rT1,off(rKP); /* k */ \ |
183 | rotrwi rT3,a,22; /* 1: S0' = a rotr 22 */ \ |
184 | evaddw w0,w0,rT0; /* w = w + s1 */ \ |
185 | xor rT2,rT2,rT3; /* 1: S0 = S0 xor S0' */ \ |
186 | evmergelohi rT0,w4,w5; /* w[-7] */ \ |
187 | and rT3,a,b; /* 1: maj = a and b */ \ |
188 | evaddw w0,w0,rT0; /* w = w + w[-7] */ \ |
189 | CMP_K##k##_LOOP \ |
190 | add rT2,rT2,rT3; /* 1: temp2 = S0 + maj */ \ |
191 | evaddw rT1,rT1,w0; /* wk = w + k */ \ |
192 | xor rT3,a,b; /* 1: maj = a xor b */ \ |
193 | evmergehi rT0,rT1,rT1; /* wk1/wk2 */ \ |
194 | and rT3,rT3,c; /* 1: maj = maj and c */ \ |
195 | add h,h,rT0; /* 1: temp1 = temp1 + wk */ \ |
196 | add rT2,rT2,rT3; /* 1: temp2 = temp2 + maj */ \ |
197 | add g,g,rT1; /* 2: temp1 = temp1 + wk */ \ |
198 | add d,d,h; /* 1: d = d + temp1 */ \ |
199 | rotrwi rT0,d,6; /* 2: S1 = e rotr 6 */ \ |
200 | add h,h,rT2; /* 1: h = temp1 + temp2 */ \ |
201 | rotrwi rT1,d,11; /* 2: S1' = e rotr 11 */ \ |
202 | rotrwi rT2,d,25; /* 2: S" = e rotr 25 */ \ |
203 | xor rT0,rT0,rT1; /* 2: S1 = S1 xor S1' */ \ |
204 | and rT3,d,e; /* 2: ch = e and f */ \ |
205 | xor rT0,rT0,rT2; /* 2: S1 = S1 xor S1" */ \ |
206 | andc rT1,f,d; /* 2: ch' = ~e and g */ \ |
207 | add g,g,rT0; /* 2: temp1 = h + S1 */ \ |
208 | xor rT3,rT3,rT1; /* 2: ch = ch xor ch' */ \ |
209 | rotrwi rT0,h,2; /* 2: S0 = a rotr 2 */ \ |
210 | add g,g,rT3; /* 2: temp1 = temp1 + ch */ \ |
211 | rotrwi rT1,h,13; /* 2: S0' = a rotr 13 */ \ |
212 | rotrwi rT3,h,22; /* 2: S0" = a rotr 22 */ \ |
213 | xor rT0,rT0,rT1; /* 2: S0 = S0 xor S0' */ \ |
214 | or rT2,h,a; /* 2: maj = a or b */ \ |
215 | and rT1,h,a; /* 2: maj' = a and b */ \ |
216 | and rT2,rT2,b; /* 2: maj = maj and c */ \ |
217 | xor rT3,rT0,rT3; /* 2: S0 = S0 xor S0" */ \ |
218 | or rT2,rT1,rT2; /* 2: maj = maj or maj' */ \ |
219 | add c,c,g; /* 2: d = d + temp1 */ \ |
220 | add rT3,rT3,rT2; /* 2: temp2 = S0 + maj */ \ |
221 | add g,g,rT3 /* 2: h = temp1 + temp2 */ |
222 | |
223 | _GLOBAL(ppc_spe_sha256_transform) |
224 | INITIALIZE |
225 | |
226 | mtctr r5 |
227 | lwz rH0,0(rHP) |
228 | lwz rH1,4(rHP) |
229 | lwz rH2,8(rHP) |
230 | lwz rH3,12(rHP) |
231 | lwz rH4,16(rHP) |
232 | lwz rH5,20(rHP) |
233 | lwz rH6,24(rHP) |
234 | lwz rH7,28(rHP) |
235 | |
236 | ppc_spe_sha256_main: |
237 | lis rKP,PPC_SPE_SHA256_K@ha |
238 | addi rKP,rKP,PPC_SPE_SHA256_K@l |
239 | |
240 | R_LOAD_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7, rW0, 0) |
241 | R_LOAD_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5, rW1, 8) |
242 | R_LOAD_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3, rW2, 16) |
243 | R_LOAD_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1, rW3, 24) |
244 | R_LOAD_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7, rW4, 32) |
245 | R_LOAD_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5, rW5, 40) |
246 | R_LOAD_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3, rW6, 48) |
247 | R_LOAD_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1, rW7, 56) |
248 | ppc_spe_sha256_16_rounds: |
249 | addi rKP,rKP,64 |
250 | R_CALC_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7, |
251 | rW0, rW1, rW4, rW5, rW7, N, 0) |
252 | R_CALC_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5, |
253 | rW1, rW2, rW5, rW6, rW0, N, 8) |
254 | R_CALC_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3, |
255 | rW2, rW3, rW6, rW7, rW1, N, 16) |
256 | R_CALC_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1, |
257 | rW3, rW4, rW7, rW0, rW2, N, 24) |
258 | R_CALC_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7, |
259 | rW4, rW5, rW0, rW1, rW3, N, 32) |
260 | R_CALC_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5, |
261 | rW5, rW6, rW1, rW2, rW4, N, 40) |
262 | R_CALC_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3, |
263 | rW6, rW7, rW2, rW3, rW5, N, 48) |
264 | R_CALC_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1, |
265 | rW7, rW0, rW3, rW4, rW6, C, 56) |
266 | bt gt,ppc_spe_sha256_16_rounds |
267 | |
268 | lwz rW0,0(rHP) |
269 | NEXT_BLOCK |
270 | lwz rW1,4(rHP) |
271 | lwz rW2,8(rHP) |
272 | lwz rW3,12(rHP) |
273 | lwz rW4,16(rHP) |
274 | lwz rW5,20(rHP) |
275 | lwz rW6,24(rHP) |
276 | lwz rW7,28(rHP) |
277 | |
278 | add rH0,rH0,rW0 |
279 | stw rH0,0(rHP) |
280 | add rH1,rH1,rW1 |
281 | stw rH1,4(rHP) |
282 | add rH2,rH2,rW2 |
283 | stw rH2,8(rHP) |
284 | add rH3,rH3,rW3 |
285 | stw rH3,12(rHP) |
286 | add rH4,rH4,rW4 |
287 | stw rH4,16(rHP) |
288 | add rH5,rH5,rW5 |
289 | stw rH5,20(rHP) |
290 | add rH6,rH6,rW6 |
291 | stw rH6,24(rHP) |
292 | add rH7,rH7,rW7 |
293 | stw rH7,28(rHP) |
294 | |
295 | bdnz ppc_spe_sha256_main |
296 | |
297 | FINALIZE |
298 | blr |
299 | |
300 | .data |
301 | .align 5 |
302 | PPC_SPE_SHA256_K: |
303 | .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 |
304 | .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 |
305 | .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 |
306 | .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 |
307 | .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc |
308 | .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da |
309 | .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 |
310 | .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 |
311 | .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 |
312 | .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 |
313 | .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 |
314 | .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 |
315 | .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 |
316 | .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 |
317 | .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 |
318 | .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 |
319 | |